2024-12-02 22:13:42,205 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 22:13:42,216 main DEBUG Took 0.008905 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 22:13:42,216 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 22:13:42,216 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 22:13:42,217 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 22:13:42,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,224 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 22:13:42,235 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,236 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,237 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,238 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,239 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,239 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,240 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,241 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,242 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,242 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,243 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,244 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,244 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 22:13:42,245 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,245 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 22:13:42,246 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 22:13:42,248 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 22:13:42,249 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 22:13:42,249 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 22:13:42,250 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 22:13:42,251 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 22:13:42,258 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 22:13:42,260 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 22:13:42,262 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 22:13:42,262 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 22:13:42,262 main DEBUG createAppenders(={Console}) 2024-12-02 22:13:42,263 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 22:13:42,263 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 22:13:42,264 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 22:13:42,264 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 22:13:42,264 main DEBUG OutputStream closed 2024-12-02 22:13:42,264 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 22:13:42,265 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 22:13:42,265 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 22:13:42,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 22:13:42,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 22:13:42,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 22:13:42,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 22:13:42,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 22:13:42,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 22:13:42,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 22:13:42,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 22:13:42,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 22:13:42,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 22:13:42,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 22:13:42,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 22:13:42,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 22:13:42,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 22:13:42,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 22:13:42,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 22:13:42,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 22:13:42,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 22:13:42,344 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 22:13:42,345 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 22:13:42,345 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 22:13:42,346 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T22:13:42,575 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3 2024-12-02 22:13:42,578 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 22:13:42,578 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T22:13:42,587 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T22:13:42,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T22:13:42,627 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85, deleteOnExit=true 2024-12-02T22:13:42,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T22:13:42,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/test.cache.data in system properties and HBase conf 2024-12-02T22:13:42,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T22:13:42,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.log.dir in system properties and HBase conf 2024-12-02T22:13:42,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T22:13:42,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T22:13:42,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T22:13:42,733 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T22:13:42,841 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T22:13:42,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T22:13:42,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T22:13:42,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T22:13:42,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T22:13:42,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T22:13:42,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T22:13:42,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T22:13:42,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T22:13:42,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T22:13:42,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/nfs.dump.dir in system properties and HBase conf 2024-12-02T22:13:42,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/java.io.tmpdir in system properties and HBase conf 2024-12-02T22:13:42,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T22:13:42,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T22:13:42,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T22:13:43,679 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T22:13:43,754 INFO [Time-limited test {}] log.Log(170): Logging initialized @2312ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T22:13:43,830 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:13:43,892 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:13:43,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:13:43,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:13:43,913 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T22:13:43,925 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:13:43,927 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:13:43,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:13:44,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/java.io.tmpdir/jetty-localhost-44573-hadoop-hdfs-3_4_1-tests_jar-_-any-10531014798575267978/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T22:13:44,126 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44573} 2024-12-02T22:13:44,126 INFO [Time-limited test {}] server.Server(415): Started @2685ms 2024-12-02T22:13:44,537 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T22:13:44,545 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T22:13:44,546 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T22:13:44,546 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T22:13:44,546 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T22:13:44,547 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.log.dir/,AVAILABLE} 2024-12-02T22:13:44,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T22:13:44,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/java.io.tmpdir/jetty-localhost-42477-hadoop-hdfs-3_4_1-tests_jar-_-any-13034434904121965320/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:13:44,671 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42477} 2024-12-02T22:13:44,671 INFO [Time-limited test {}] server.Server(415): Started @3230ms 2024-12-02T22:13:44,727 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T22:13:45,209 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data1/current/BP-1855264631-172.17.0.2-1733177623427/current, will proceed with Du for space computation calculation, 2024-12-02T22:13:45,209 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data2/current/BP-1855264631-172.17.0.2-1733177623427/current, will proceed with Du for space computation calculation, 2024-12-02T22:13:45,252 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T22:13:45,311 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x89cd731fdc1e0b4b with lease ID 0x7f2615b4267d8ef9: Processing first storage report for DS-4982d8d7-a8eb-477c-80d8-bf1dd2e5e94a from datanode DatanodeRegistration(127.0.0.1:46173, datanodeUuid=822023b4-62d7-49d7-a9ba-af97b603c81e, infoPort=46595, infoSecurePort=0, ipcPort=37903, storageInfo=lv=-57;cid=testClusterID;nsid=767598192;c=1733177623427) 2024-12-02T22:13:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89cd731fdc1e0b4b with lease ID 0x7f2615b4267d8ef9: from storage DS-4982d8d7-a8eb-477c-80d8-bf1dd2e5e94a node DatanodeRegistration(127.0.0.1:46173, datanodeUuid=822023b4-62d7-49d7-a9ba-af97b603c81e, infoPort=46595, infoSecurePort=0, ipcPort=37903, storageInfo=lv=-57;cid=testClusterID;nsid=767598192;c=1733177623427), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T22:13:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x89cd731fdc1e0b4b with lease ID 0x7f2615b4267d8ef9: Processing first storage report for DS-513e95c1-d275-4e80-a243-a868362e3832 from datanode DatanodeRegistration(127.0.0.1:46173, datanodeUuid=822023b4-62d7-49d7-a9ba-af97b603c81e, infoPort=46595, infoSecurePort=0, ipcPort=37903, storageInfo=lv=-57;cid=testClusterID;nsid=767598192;c=1733177623427) 2024-12-02T22:13:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89cd731fdc1e0b4b with lease ID 0x7f2615b4267d8ef9: from storage DS-513e95c1-d275-4e80-a243-a868362e3832 node DatanodeRegistration(127.0.0.1:46173, datanodeUuid=822023b4-62d7-49d7-a9ba-af97b603c81e, infoPort=46595, infoSecurePort=0, ipcPort=37903, storageInfo=lv=-57;cid=testClusterID;nsid=767598192;c=1733177623427), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T22:13:45,398 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3 2024-12-02T22:13:45,484 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/zookeeper_0, clientPort=58452, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T22:13:45,495 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58452 2024-12-02T22:13:45,507 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:45,511 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:45,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741825_1001 (size=7) 2024-12-02T22:13:46,164 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c with version=8 2024-12-02T22:13:46,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/hbase-staging 2024-12-02T22:13:46,257 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T22:13:46,534 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:13:46,549 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:46,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:46,554 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:13:46,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:46,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:13:46,692 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T22:13:46,753 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T22:13:46,763 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T22:13:46,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:13:46,793 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3066 (auto-detected) 2024-12-02T22:13:46,794 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T22:13:46,816 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36511 2024-12-02T22:13:46,844 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36511 connecting to ZooKeeper ensemble=127.0.0.1:58452 2024-12-02T22:13:46,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365110x0, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:13:46,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36511-0x1009d0dd6140000 connected 2024-12-02T22:13:46,910 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:46,912 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:46,921 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:13:46,925 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c, hbase.cluster.distributed=false 2024-12-02T22:13:46,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:13:46,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36511 2024-12-02T22:13:46,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36511 2024-12-02T22:13:46,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36511 2024-12-02T22:13:46,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36511 2024-12-02T22:13:46,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36511 2024-12-02T22:13:47,095 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7ae73638d47e:0 server-side Connection retries=45 2024-12-02T22:13:47,097 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:47,098 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:47,098 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T22:13:47,098 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T22:13:47,098 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T22:13:47,101 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T22:13:47,104 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T22:13:47,105 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41753 2024-12-02T22:13:47,107 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41753 connecting to ZooKeeper ensemble=127.0.0.1:58452 2024-12-02T22:13:47,109 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:47,112 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:47,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417530x0, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T22:13:47,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417530x0, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:13:47,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41753-0x1009d0dd6140001 connected 2024-12-02T22:13:47,124 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T22:13:47,133 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T22:13:47,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T22:13:47,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T22:13:47,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-02T22:13:47,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41753 2024-12-02T22:13:47,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41753 2024-12-02T22:13:47,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-02T22:13:47,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-02T22:13:47,162 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7ae73638d47e:36511 2024-12-02T22:13:47,163 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:13:47,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:13:47,171 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T22:13:47,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,191 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T22:13:47,192 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7ae73638d47e,36511,1733177626309 from backup master directory 2024-12-02T22:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T22:13:47,195 WARN [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:13:47,196 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,198 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T22:13:47,199 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T22:13:47,252 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/hbase.id] with ID: ffa625a2-8222-4ade-8d4e-7a91ac6a7448 2024-12-02T22:13:47,252 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/.tmp/hbase.id 2024-12-02T22:13:47,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741826_1002 (size=42) 2024-12-02T22:13:47,263 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/.tmp/hbase.id]:[hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/hbase.id] 2024-12-02T22:13:47,307 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T22:13:47,313 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T22:13:47,334 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-02T22:13:47,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741827_1003 (size=196) 2024-12-02T22:13:47,371 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:13:47,374 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T22:13:47,391 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:47,395 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:13:47,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741828_1004 (size=1189) 2024-12-02T22:13:47,443 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store 2024-12-02T22:13:47,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741829_1005 (size=34) 2024-12-02T22:13:47,469 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T22:13:47,473 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:47,475 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T22:13:47,475 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:13:47,475 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:13:47,477 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T22:13:47,477 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:13:47,477 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:13:47,478 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733177627474Disabling compacts and flushes for region at 1733177627474Disabling writes for close at 1733177627477 (+3 ms)Writing region close event to WAL at 1733177627477Closed at 1733177627477 2024-12-02T22:13:47,481 WARN [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/.initializing 2024-12-02T22:13:47,481 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/WALs/7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,491 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:13:47,506 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C36511%2C1733177626309, suffix=, logDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/WALs/7ae73638d47e,36511,1733177626309, archiveDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/oldWALs, maxLogs=10 2024-12-02T22:13:47,540 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/WALs/7ae73638d47e,36511,1733177626309/7ae73638d47e%2C36511%2C1733177626309.1733177627511, exclude list is [], retry=0 2024-12-02T22:13:47,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46173,DS-4982d8d7-a8eb-477c-80d8-bf1dd2e5e94a,DISK] 2024-12-02T22:13:47,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T22:13:47,610 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/WALs/7ae73638d47e,36511,1733177626309/7ae73638d47e%2C36511%2C1733177626309.1733177627511 2024-12-02T22:13:47,611 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46595:46595)] 2024-12-02T22:13:47,612 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:13:47,613 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:47,617 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,618 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T22:13:47,685 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:47,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:47,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T22:13:47,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:47,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:47,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T22:13:47,696 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:47,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:47,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T22:13:47,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:47,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:47,701 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,705 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,706 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,711 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,712 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,716 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T22:13:47,720 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T22:13:47,724 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:13:47,725 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69086584, jitterRate=0.029470324516296387}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T22:13:47,733 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733177627631Initializing all the Stores at 1733177627633 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177627634 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177627634Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177627635 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177627635Cleaning up temporary data from old regions at 1733177627712 (+77 ms)Region opened successfully at 1733177627732 (+20 ms) 2024-12-02T22:13:47,734 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T22:13:47,773 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eed2c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:13:47,806 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T22:13:47,819 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T22:13:47,819 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T22:13:47,822 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T22:13:47,824 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T22:13:47,829 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T22:13:47,829 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T22:13:47,855 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T22:13:47,867 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T22:13:47,869 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T22:13:47,872 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T22:13:47,874 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T22:13:47,876 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T22:13:47,878 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T22:13:47,881 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T22:13:47,883 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T22:13:47,884 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T22:13:47,886 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T22:13:47,904 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T22:13:47,906 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T22:13:47,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:13:47,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T22:13:47,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,913 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7ae73638d47e,36511,1733177626309, sessionid=0x1009d0dd6140000, setting cluster-up flag (Was=false) 2024-12-02T22:13:47,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,931 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T22:13:47,933 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:47,944 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T22:13:47,945 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7ae73638d47e,36511,1733177626309 2024-12-02T22:13:47,952 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T22:13:48,028 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T22:13:48,039 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T22:13:48,047 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T22:13:48,050 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(746): ClusterId : ffa625a2-8222-4ade-8d4e-7a91ac6a7448 2024-12-02T22:13:48,053 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T22:13:48,053 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7ae73638d47e,36511,1733177626309 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T22:13:48,059 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T22:13:48,059 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T22:13:48,061 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:13:48,061 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:13:48,061 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:13:48,062 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7ae73638d47e:0, corePoolSize=5, maxPoolSize=5 2024-12-02T22:13:48,062 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7ae73638d47e:0, corePoolSize=10, maxPoolSize=10 2024-12-02T22:13:48,062 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T22:13:48,062 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,062 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:13:48,063 DEBUG [RS:0;7ae73638d47e:41753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cce18ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7ae73638d47e/172.17.0.2:0 2024-12-02T22:13:48,063 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,065 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733177658065 2024-12-02T22:13:48,067 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T22:13:48,068 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T22:13:48,071 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:13:48,071 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T22:13:48,073 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T22:13:48,073 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T22:13:48,074 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T22:13:48,074 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T22:13:48,075 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,078 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T22:13:48,079 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T22:13:48,079 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:48,080 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T22:13:48,079 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T22:13:48,082 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7ae73638d47e:41753 2024-12-02T22:13:48,084 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T22:13:48,085 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T22:13:48,085 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T22:13:48,085 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T22:13:48,085 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T22:13:48,088 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(2659): reportForDuty to master=7ae73638d47e,36511,1733177626309 with port=41753, startcode=1733177627056 2024-12-02T22:13:48,088 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733177628086,5,FailOnTimeoutGroup] 2024-12-02T22:13:48,090 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733177628088,5,FailOnTimeoutGroup] 2024-12-02T22:13:48,090 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,091 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T22:13:48,092 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,093 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741831_1007 (size=1321) 2024-12-02T22:13:48,097 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T22:13:48,097 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:13:48,101 DEBUG [RS:0;7ae73638d47e:41753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T22:13:48,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741832_1008 (size=32) 2024-12-02T22:13:48,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42805, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T22:13:48,146 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36511 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,148 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36511 {}] master.ServerManager(517): Registering regionserver=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,165 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:13:48,165 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34911 2024-12-02T22:13:48,165 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T22:13:48,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:13:48,170 DEBUG [RS:0;7ae73638d47e:41753 {}] zookeeper.ZKUtil(111): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,171 WARN [RS:0;7ae73638d47e:41753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T22:13:48,171 INFO [RS:0;7ae73638d47e:41753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:13:48,171 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7ae73638d47e,41753,1733177627056] 2024-12-02T22:13:48,196 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T22:13:48,209 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T22:13:48,214 INFO [RS:0;7ae73638d47e:41753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T22:13:48,214 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,215 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T22:13:48,221 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T22:13:48,222 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,222 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,222 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,223 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,223 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,223 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,223 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7ae73638d47e:0, corePoolSize=2, maxPoolSize=2 2024-12-02T22:13:48,224 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,224 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,224 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,224 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,225 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,225 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7ae73638d47e:0, corePoolSize=1, maxPoolSize=1 2024-12-02T22:13:48,225 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:13:48,225 DEBUG [RS:0;7ae73638d47e:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0, corePoolSize=3, maxPoolSize=3 2024-12-02T22:13:48,227 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,227 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,227 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,227 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,228 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,228 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,41753,1733177627056-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:13:48,247 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T22:13:48,249 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,41753,1733177627056-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,249 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,249 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.Replication(171): 7ae73638d47e,41753,1733177627056 started 2024-12-02T22:13:48,268 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:48,268 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1482): Serving as 7ae73638d47e,41753,1733177627056, RpcServer on 7ae73638d47e/172.17.0.2:41753, sessionid=0x1009d0dd6140001 2024-12-02T22:13:48,269 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T22:13:48,269 DEBUG [RS:0;7ae73638d47e:41753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,269 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,41753,1733177627056' 2024-12-02T22:13:48,269 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T22:13:48,270 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T22:13:48,271 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T22:13:48,271 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T22:13:48,271 DEBUG [RS:0;7ae73638d47e:41753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,271 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7ae73638d47e,41753,1733177627056' 2024-12-02T22:13:48,271 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T22:13:48,272 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T22:13:48,272 DEBUG [RS:0;7ae73638d47e:41753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T22:13:48,272 INFO [RS:0;7ae73638d47e:41753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T22:13:48,272 INFO [RS:0;7ae73638d47e:41753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T22:13:48,377 INFO [RS:0;7ae73638d47e:41753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T22:13:48,380 INFO [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C41753%2C1733177627056, suffix=, logDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056, archiveDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/oldWALs, maxLogs=32 2024-12-02T22:13:48,396 DEBUG [RS:0;7ae73638d47e:41753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056/7ae73638d47e%2C41753%2C1733177627056.1733177628382, exclude list is [], retry=0 2024-12-02T22:13:48,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46173,DS-4982d8d7-a8eb-477c-80d8-bf1dd2e5e94a,DISK] 2024-12-02T22:13:48,404 INFO [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056/7ae73638d47e%2C41753%2C1733177627056.1733177628382 2024-12-02T22:13:48,404 DEBUG [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46595:46595)] 2024-12-02T22:13:48,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:48,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T22:13:48,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T22:13:48,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:48,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:48,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T22:13:48,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T22:13:48,520 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:48,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:48,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T22:13:48,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T22:13:48,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:48,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:48,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T22:13:48,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T22:13:48,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:48,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:48,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T22:13:48,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740 2024-12-02T22:13:48,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740 2024-12-02T22:13:48,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T22:13:48,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T22:13:48,535 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:13:48,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T22:13:48,541 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:13:48,542 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70493407, jitterRate=0.05043362081050873}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:13:48,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733177628509Initializing all the Stores at 1733177628511 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177628511Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177628512 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177628512Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177628512Cleaning up temporary data from old regions at 1733177628534 (+22 ms)Region opened successfully at 1733177628545 (+11 ms) 2024-12-02T22:13:48,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T22:13:48,546 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T22:13:48,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T22:13:48,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T22:13:48,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T22:13:48,548 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T22:13:48,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733177628546Disabling compacts and flushes for region at 1733177628546Disabling writes for close at 1733177628546Writing region close event to WAL at 1733177628547 (+1 ms)Closed at 1733177628548 (+1 ms) 2024-12-02T22:13:48,552 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:13:48,553 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T22:13:48,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T22:13:48,568 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:13:48,571 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T22:13:48,723 DEBUG [7ae73638d47e:36511 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T22:13:48,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:48,744 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,41753,1733177627056, state=OPENING 2024-12-02T22:13:48,749 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T22:13:48,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:48,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:13:48,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:13:48,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:13:48,753 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T22:13:48,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:13:48,934 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T22:13:48,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T22:13:48,949 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T22:13:48,950 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T22:13:48,950 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T22:13:48,954 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7ae73638d47e%2C41753%2C1733177627056.meta, suffix=.meta, logDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056, archiveDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/oldWALs, maxLogs=32 2024-12-02T22:13:48,972 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056/7ae73638d47e%2C41753%2C1733177627056.meta.1733177628956.meta, exclude list is [], retry=0 2024-12-02T22:13:48,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46173,DS-4982d8d7-a8eb-477c-80d8-bf1dd2e5e94a,DISK] 2024-12-02T22:13:48,981 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/WALs/7ae73638d47e,41753,1733177627056/7ae73638d47e%2C41753%2C1733177627056.meta.1733177628956.meta 2024-12-02T22:13:48,982 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46595:46595)] 2024-12-02T22:13:48,982 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:13:48,984 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T22:13:48,987 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T22:13:48,992 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T22:13:48,997 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T22:13:48,998 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:48,998 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T22:13:48,998 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T22:13:49,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T22:13:49,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T22:13:49,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:49,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:49,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T22:13:49,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T22:13:49,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:49,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:49,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T22:13:49,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T22:13:49,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:49,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:49,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T22:13:49,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T22:13:49,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:49,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T22:13:49,013 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T22:13:49,014 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740 2024-12-02T22:13:49,017 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740 2024-12-02T22:13:49,020 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T22:13:49,020 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T22:13:49,021 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:13:49,024 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T22:13:49,026 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59036260, jitterRate=-0.12029117345809937}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:13:49,026 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T22:13:49,027 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733177628999Writing region info on filesystem at 1733177628999Initializing all the Stores at 1733177629001 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177629001Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177629001Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177629001Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733177629001Cleaning up temporary data from old regions at 1733177629020 (+19 ms)Running coprocessor post-open hooks at 1733177629026 (+6 ms)Region opened successfully at 1733177629027 (+1 ms) 2024-12-02T22:13:49,035 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733177628923 2024-12-02T22:13:49,048 DEBUG [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T22:13:49,048 INFO [RS_OPEN_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T22:13:49,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:49,052 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7ae73638d47e,41753,1733177627056, state=OPEN 2024-12-02T22:13:49,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:13:49,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T22:13:49,058 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:13:49,058 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T22:13:49,058 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:49,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T22:13:49,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7ae73638d47e,41753,1733177627056 in 305 msec 2024-12-02T22:13:49,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T22:13:49,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 507 msec 2024-12-02T22:13:49,072 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T22:13:49,072 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T22:13:49,094 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:49,096 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:49,118 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:49,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46297, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:49,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1630 sec 2024-12-02T22:13:49,146 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733177629146, completionTime=-1 2024-12-02T22:13:49,150 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T22:13:49,150 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T22:13:49,180 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T22:13:49,180 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733177689180 2024-12-02T22:13:49,180 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733177749180 2024-12-02T22:13:49,180 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-02T22:13:49,183 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,183 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,183 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,185 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7ae73638d47e:36511, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,185 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,185 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,191 DEBUG [master/7ae73638d47e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T22:13:49,212 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.016sec 2024-12-02T22:13:49,213 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T22:13:49,215 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T22:13:49,216 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T22:13:49,216 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T22:13:49,216 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T22:13:49,217 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T22:13:49,218 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T22:13:49,226 DEBUG [master/7ae73638d47e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T22:13:49,227 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T22:13:49,228 INFO [master/7ae73638d47e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7ae73638d47e,36511,1733177626309-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T22:13:49,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:49,262 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T22:13:49,263 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T22:13:49,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:49,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:49,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17703dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:49,286 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:49,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:49,290 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:49,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:49,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7ae73638d47e,36511,1733177626309 2024-12-02T22:13:49,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=8416 2024-12-02T22:13:49,341 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:13:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:13:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55eec305 2024-12-02T22:13:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:13:49,383 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:13:49,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:13:49,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:13:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:13:49,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:13:49,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T22:13:49,411 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:49,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:13:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:49,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741835_1011 (size=1061) 2024-12-02T22:13:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:49,837 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:13:49,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741836_1012 (size=53) 2024-12-02T22:13:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:50,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:50,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8de9171fd3e9b964cfb173aae742ae78, disabling compactions & flushes 2024-12-02T22:13:50,250 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. after waiting 0 ms 2024-12-02T22:13:50,250 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,251 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,251 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8de9171fd3e9b964cfb173aae742ae78: Waiting for close lock at 1733177630250Disabling compacts and flushes for region at 1733177630250Disabling writes for close at 1733177630250Writing region close event to WAL at 1733177630250Closed at 1733177630250 2024-12-02T22:13:50,253 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:13:50,259 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177630253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177630253"}]},"ts":"1733177630253"} 2024-12-02T22:13:50,265 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:13:50,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:13:50,270 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177630267"}]},"ts":"1733177630267"} 2024-12-02T22:13:50,275 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:13:50,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, ASSIGN}] 2024-12-02T22:13:50,280 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, ASSIGN 2024-12-02T22:13:50,281 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:13:50,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8de9171fd3e9b964cfb173aae742ae78, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:50,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, ASSIGN because future has completed 2024-12-02T22:13:50,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:13:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:50,602 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,602 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:13:50,602 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,602 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:13:50,603 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,603 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,605 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,609 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:13:50,609 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8de9171fd3e9b964cfb173aae742ae78 columnFamilyName A 2024-12-02T22:13:50,609 DEBUG [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:50,610 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(327): Store=8de9171fd3e9b964cfb173aae742ae78/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:50,611 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,613 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:13:50,613 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8de9171fd3e9b964cfb173aae742ae78 columnFamilyName B 2024-12-02T22:13:50,613 DEBUG [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:50,614 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(327): Store=8de9171fd3e9b964cfb173aae742ae78/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:50,614 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,616 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:13:50,617 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8de9171fd3e9b964cfb173aae742ae78 columnFamilyName C 2024-12-02T22:13:50,617 DEBUG [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:13:50,618 INFO [StoreOpener-8de9171fd3e9b964cfb173aae742ae78-1 {}] regionserver.HStore(327): Store=8de9171fd3e9b964cfb173aae742ae78/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:13:50,618 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,619 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,620 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,621 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,623 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,623 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,624 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:13:50,627 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,631 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:13:50,632 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8de9171fd3e9b964cfb173aae742ae78; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62786086, jitterRate=-0.06441441178321838}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:13:50,632 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:50,634 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8de9171fd3e9b964cfb173aae742ae78: Running coprocessor pre-open hook at 1733177630603Writing region info on filesystem at 1733177630603Initializing all the Stores at 1733177630605 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177630605Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177630605Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177630605Cleaning up temporary data from old regions at 1733177630623 (+18 ms)Running coprocessor post-open hooks at 1733177630632 (+9 ms)Region opened successfully at 1733177630634 (+2 ms) 2024-12-02T22:13:50,636 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., pid=6, masterSystemTime=1733177630595 2024-12-02T22:13:50,640 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,640 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:50,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8de9171fd3e9b964cfb173aae742ae78, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:50,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:13:50,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T22:13:50,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 in 207 msec 2024-12-02T22:13:50,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T22:13:50,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, ASSIGN in 374 msec 2024-12-02T22:13:50,657 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:13:50,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177630657"}]},"ts":"1733177630657"} 2024-12-02T22:13:50,660 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:13:50,662 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:13:50,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-12-02T22:13:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T22:13:51,575 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:13:51,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,585 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29446fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e077d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,593 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:13:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:13:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d4e458b 2024-12-02T22:13:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:13:51,611 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:13:51,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,626 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,632 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e412ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfbcabb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,697 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876c349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429147d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16359f4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,707 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44731cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870009d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d95629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3837915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:13:51,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:13:51,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:13:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:13:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:13:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e873e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:13:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:13:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:13:51,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:13:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:13:51,738 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,750 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,770 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,793 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,796 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,809 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,823 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,834 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,862 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:13:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:13:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:13:51,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:13:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:13:51,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:13:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T22:13:51,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:13:51,904 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:13:51,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:13:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:51,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:13:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:51,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:51,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:51,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:51,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:52,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/d610823a67d84135b15adc57840aa4fc is 50, key is test_row_0/A:col10/1733177631897/Put/seqid=0 2024-12-02T22:13:52,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177692107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177692117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177692124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177692128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741837_1013 (size=12001) 2024-12-02T22:13:52,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177692147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177692365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177692365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177692366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177692375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177692366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177692583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177692594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177692605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177692610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/d610823a67d84135b15adc57840aa4fc 2024-12-02T22:13:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177692611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/eec0a457ea454be586e946184ad40eca is 50, key is test_row_0/B:col10/1733177631897/Put/seqid=0 2024-12-02T22:13:52,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741838_1014 (size=12001) 2024-12-02T22:13:52,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177692904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177692914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177692925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177692933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177692934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:52,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:52,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:52,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:52,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:53,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:53,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/eec0a457ea454be586e946184ad40eca 2024-12-02T22:13:53,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/99bc828807764794a7aba8cac425322a is 50, key is test_row_0/C:col10/1733177631897/Put/seqid=0 2024-12-02T22:13:53,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741839_1015 (size=12001) 2024-12-02T22:13:53,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177693413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177693434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:53,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177693444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177693452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:53,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177693454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:53,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/99bc828807764794a7aba8cac425322a 2024-12-02T22:13:53,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/d610823a67d84135b15adc57840aa4fc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc 2024-12-02T22:13:53,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T22:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/eec0a457ea454be586e946184ad40eca as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca 2024-12-02T22:13:53,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:53,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T22:13:53,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/99bc828807764794a7aba8cac425322a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a 2024-12-02T22:13:53,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T22:13:53,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8de9171fd3e9b964cfb173aae742ae78 in 1886ms, sequenceid=13, compaction requested=false 2024-12-02T22:13:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T22:13:53,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:53,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T22:13:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:53,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/ad9998f1909e465faf555ae59b94cd06 is 50, key is test_row_0/A:col10/1733177632125/Put/seqid=0 2024-12-02T22:13:54,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741840_1016 (size=12001) 2024-12-02T22:13:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:54,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/ad9998f1909e465faf555ae59b94cd06 2024-12-02T22:13:54,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:54,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/df834ccc12544d8c9930c5f0a6d39950 is 50, key is test_row_0/B:col10/1733177632125/Put/seqid=0 2024-12-02T22:13:54,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741841_1017 (size=12001) 2024-12-02T22:13:54,482 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/df834ccc12544d8c9930c5f0a6d39950 2024-12-02T22:13:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177694481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177694476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177694482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177694493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,500 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2392 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,502 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177694496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/e0695668db9a4d728eb81ea61bb22730 is 50, key is test_row_0/C:col10/1733177632125/Put/seqid=0 2024-12-02T22:13:54,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741842_1018 (size=12001) 2024-12-02T22:13:54,595 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/e0695668db9a4d728eb81ea61bb22730 2024-12-02T22:13:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177694604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177694604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/ad9998f1909e465faf555ae59b94cd06 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06 2024-12-02T22:13:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177694614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177694614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T22:13:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/df834ccc12544d8c9930c5f0a6d39950 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950 2024-12-02T22:13:54,651 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T22:13:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/e0695668db9a4d728eb81ea61bb22730 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730 2024-12-02T22:13:54,677 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T22:13:54,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8de9171fd3e9b964cfb173aae742ae78 in 747ms, sequenceid=33, compaction requested=false 2024-12-02T22:13:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T22:13:54,683 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T22:13:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T22:13:54,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T22:13:54,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7820 sec 2024-12-02T22:13:54,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8060 sec 2024-12-02T22:13:54,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:54,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/c5c0ceb5f4f24e36bed59546f873402a is 50, key is test_row_0/A:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741843_1019 (size=12001) 2024-12-02T22:13:54,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/c5c0ceb5f4f24e36bed59546f873402a 2024-12-02T22:13:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177694879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177694884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177694889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177694892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:54,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/4c765d694d7c40608057c8589605d175 is 50, key is test_row_0/B:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:54,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741844_1020 (size=12001) 2024-12-02T22:13:54,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/4c765d694d7c40608057c8589605d175 2024-12-02T22:13:54,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/2d49a101e111450d84ed0a90e6a6a2dd is 50, key is test_row_0/C:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177695004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177695005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177695005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741845_1021 (size=12001) 2024-12-02T22:13:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177695013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177695214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177695214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177695224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177695245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:55,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/2d49a101e111450d84ed0a90e6a6a2dd 2024-12-02T22:13:55,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/c5c0ceb5f4f24e36bed59546f873402a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a 2024-12-02T22:13:55,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T22:13:55,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/4c765d694d7c40608057c8589605d175 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175 2024-12-02T22:13:55,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T22:13:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/2d49a101e111450d84ed0a90e6a6a2dd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd 2024-12-02T22:13:55,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T22:13:55,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8de9171fd3e9b964cfb173aae742ae78 in 651ms, sequenceid=47, compaction requested=true 2024-12-02T22:13:55,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:13:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:13:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:55,491 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:13:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:13:55,493 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:55,497 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:55,499 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:13:55,500 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:55,500 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.2 K 2024-12-02T22:13:55,502 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting eec0a457ea454be586e946184ad40eca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733177631894 2024-12-02T22:13:55,502 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting df834ccc12544d8c9930c5f0a6d39950, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733177632074 2024-12-02T22:13:55,503 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:55,503 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:13:55,503 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c765d694d7c40608057c8589605d175, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:55,504 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:55,505 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.2 K 2024-12-02T22:13:55,507 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d610823a67d84135b15adc57840aa4fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733177631894 2024-12-02T22:13:55,508 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9998f1909e465faf555ae59b94cd06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733177632074 2024-12-02T22:13:55,509 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c0ceb5f4f24e36bed59546f873402a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:55,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:13:55,560 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:55,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177634874, max timestamp=1733177635545Num uniques 859; 2024-12-02T22:13:55,561 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a4049a7ea3594ba4ac96a094ad52de67 is 50, key is test_row_0/A:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:55,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,565 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#9 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:55,566 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/d791b9ba5819476dbeed3cb0eded0e32 is 50, key is test_row_0/B:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177634874, max timestamp=1733177635545Num uniques 859; 2024-12-02T22:13:55,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:55,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177634874, max timestamp=1733177635545Num uniques 859; 2024-12-02T22:13:55,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741847_1023 (size=12104) 2024-12-02T22:13:55,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1da87f06eefa4da98c37c12a7ca282cf is 50, key is test_row_0/A:col10/1733177635545/Put/seqid=0 2024-12-02T22:13:55,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741846_1022 (size=12104) 2024-12-02T22:13:55,628 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/d791b9ba5819476dbeed3cb0eded0e32 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d791b9ba5819476dbeed3cb0eded0e32 2024-12-02T22:13:55,640 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a4049a7ea3594ba4ac96a094ad52de67 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a4049a7ea3594ba4ac96a094ad52de67 2024-12-02T22:13:55,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741848_1024 (size=12001) 2024-12-02T22:13:55,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1da87f06eefa4da98c37c12a7ca282cf 2024-12-02T22:13:55,667 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into a4049a7ea3594ba4ac96a094ad52de67(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:55,667 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into d791b9ba5819476dbeed3cb0eded0e32(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:55,667 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:55,667 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:55,671 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177635479; duration=0sec 2024-12-02T22:13:55,671 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177635491; duration=0sec 2024-12-02T22:13:55,671 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:55,672 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:13:55,672 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:55,674 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:55,674 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:13:55,675 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:55,676 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:13:55,676 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:55,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6065d71d30c34f8baab2077d3f75c4bd is 50, key is test_row_0/B:col10/1733177635545/Put/seqid=0 2024-12-02T22:13:55,676 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.2 K 2024-12-02T22:13:55,678 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bc828807764794a7aba8cac425322a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733177631894 2024-12-02T22:13:55,679 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e0695668db9a4d728eb81ea61bb22730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733177632074 2024-12-02T22:13:55,680 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d49a101e111450d84ed0a90e6a6a2dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:55,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741849_1025 (size=12001) 2024-12-02T22:13:55,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6065d71d30c34f8baab2077d3f75c4bd 2024-12-02T22:13:55,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177635657, max timestamp=1733177635688Num uniques 859; 2024-12-02T22:13:55,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,707 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:55,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/672a9a44898b4c5d874fc73959ab8ebc is 50, key is test_row_0/C:col10/1733177634819/Put/seqid=0 2024-12-02T22:13:55,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177635657, max timestamp=1733177635691Num uniques 859; 2024-12-02T22:13:55,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/67362084d7074bdbb25d26ec28bab517 is 50, key is test_row_0/C:col10/1733177635545/Put/seqid=0 2024-12-02T22:13:55,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177635657, max timestamp=1733177635688Num uniques 859; 2024-12-02T22:13:55,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741851_1027 (size=12001) 2024-12-02T22:13:55,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/67362084d7074bdbb25d26ec28bab517 2024-12-02T22:13:55,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1da87f06eefa4da98c37c12a7ca282cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf 2024-12-02T22:13:55,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741850_1026 (size=12104) 2024-12-02T22:13:55,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T22:13:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6065d71d30c34f8baab2077d3f75c4bd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd 2024-12-02T22:13:55,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177635781, max timestamp=1733177635802Num uniques 1018; 2024-12-02T22:13:55,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T22:13:55,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T22:13:55,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/67362084d7074bdbb25d26ec28bab517 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517 2024-12-02T22:13:55,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177635781, max timestamp=1733177635802Num uniques 1018; 2024-12-02T22:13:55,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T22:13:55,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177635781, max timestamp=1733177635802Num uniques 1018; 2024-12-02T22:13:55,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T22:13:55,838 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/672a9a44898b4c5d874fc73959ab8ebc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/672a9a44898b4c5d874fc73959ab8ebc 2024-12-02T22:13:55,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T22:13:55,851 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into 672a9a44898b4c5d874fc73959ab8ebc(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:55,851 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:55,852 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177635491; duration=0sec 2024-12-02T22:13:55,852 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:55,852 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:13:55,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=53.67 KB/54960 for 8de9171fd3e9b964cfb173aae742ae78 in 308ms, sequenceid=68, compaction requested=false 2024-12-02T22:13:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:55,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=60.38 KB heapSize=165 KB 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:55,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/62237694ca5c481d8987f74db3447d69 is 50, key is test_row_0/A:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741852_1028 (size=12001) 2024-12-02T22:13:55,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177635908, max timestamp=1733177635940Num uniques 859; 2024-12-02T22:13:55,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:13:55,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177635908, max timestamp=1733177635940Num uniques 859; 2024-12-02T22:13:55,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:13:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733177635908, max timestamp=1733177635935Num uniques 859; 2024-12-02T22:13:55,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T22:13:56,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177695990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177696000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177696003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177696008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T22:13:56,064 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:13:56,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:13:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T22:13:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T22:13:56,071 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:13:56,073 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:13:56,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:13:56,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177696113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177696124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177696124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177696124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T22:13:56,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T22:13:56,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/62237694ca5c481d8987f74db3447d69 2024-12-02T22:13:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177696323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177696334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3befd9e48f0f40b88ad3e8660776eb27 is 50, key is test_row_0/B:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177696343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177696343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T22:13:56,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T22:13:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741853_1029 (size=12001) 2024-12-02T22:13:56,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3befd9e48f0f40b88ad3e8660776eb27 2024-12-02T22:13:56,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/06128674cb564c238c893db21d4ff556 is 50, key is test_row_0/C:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:56,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741854_1030 (size=12001) 2024-12-02T22:13:56,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/06128674cb564c238c893db21d4ff556 2024-12-02T22:13:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/62237694ca5c481d8987f74db3447d69 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69 2024-12-02T22:13:56,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T22:13:56,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3befd9e48f0f40b88ad3e8660776eb27 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27 2024-12-02T22:13:56,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T22:13:56,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/06128674cb564c238c893db21d4ff556 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556 2024-12-02T22:13:56,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T22:13:56,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177696514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,523 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4418 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:13:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,525 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T22:13:56,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=122.60 KB/125545 for 8de9171fd3e9b964cfb173aae742ae78 in 669ms, sequenceid=114, compaction requested=true 2024-12-02T22:13:56,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:13:56,527 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:13:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:56,527 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:56,530 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:56,530 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:13:56,530 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,531 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d791b9ba5819476dbeed3cb0eded0e32, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.3 K 2024-12-02T22:13:56,532 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:56,532 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:13:56,532 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,532 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a4049a7ea3594ba4ac96a094ad52de67, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.3 K 2024-12-02T22:13:56,533 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d791b9ba5819476dbeed3cb0eded0e32, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:56,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4049a7ea3594ba4ac96a094ad52de67, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:56,534 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6065d71d30c34f8baab2077d3f75c4bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733177634874 2024-12-02T22:13:56,535 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da87f06eefa4da98c37c12a7ca282cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733177634874 2024-12-02T22:13:56,536 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3befd9e48f0f40b88ad3e8660776eb27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:56,537 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62237694ca5c481d8987f74db3447d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:56,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T22:13:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,553 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=122.60 KB heapSize=349.30 KB 2024-12-02T22:13:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:13:56,573 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/fa969eefb40e437bb413bacbfa3067e1 is 50, key is test_row_0/A:col10/1733177635978/Put/seqid=0 2024-12-02T22:13:56,574 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/d303102ca3a548fbb591dbbe0edea178 is 50, key is test_row_0/B:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:56,576 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#18 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:56,577 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f6fe5b934a6d4dea81b053acd884e923 is 50, key is test_row_0/A:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:56,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741855_1031 (size=12207) 2024-12-02T22:13:56,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741857_1033 (size=12207) 2024-12-02T22:13:56,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741856_1032 (size=12151) 2024-12-02T22:13:56,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/fa969eefb40e437bb413bacbfa3067e1 2024-12-02T22:13:56,630 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f6fe5b934a6d4dea81b053acd884e923 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f6fe5b934a6d4dea81b053acd884e923 2024-12-02T22:13:56,644 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into f6fe5b934a6d4dea81b053acd884e923(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:56,645 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:56,645 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177636527; duration=0sec 2024-12-02T22:13:56,645 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:56,645 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:13:56,645 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:56,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/76fc0e5972ed4b578373a499ee3af6c1 is 50, key is test_row_0/B:col10/1733177635978/Put/seqid=0 2024-12-02T22:13:56,649 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:56,649 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:13:56,649 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:56,649 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/672a9a44898b4c5d874fc73959ab8ebc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.3 K 2024-12-02T22:13:56,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:56,651 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 672a9a44898b4c5d874fc73959ab8ebc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733177634480 2024-12-02T22:13:56,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67362084d7074bdbb25d26ec28bab517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733177634874 2024-12-02T22:13:56,655 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06128674cb564c238c893db21d4ff556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:56,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741858_1034 (size=12151) 2024-12-02T22:13:56,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/76fc0e5972ed4b578373a499ee3af6c1 2024-12-02T22:13:56,690 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#22 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:56,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177696679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177696679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,691 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/d62f069c4c0e4a4796d93dfab2d9a605 is 50, key is test_row_0/C:col10/1733177635847/Put/seqid=0 2024-12-02T22:13:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T22:13:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177696682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177696687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/0eb2706fe47b43a698413b8f9608ec4e is 50, key is test_row_0/C:col10/1733177635978/Put/seqid=0 2024-12-02T22:13:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741860_1036 (size=12151) 2024-12-02T22:13:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741859_1035 (size=12207) 2024-12-02T22:13:56,734 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/d62f069c4c0e4a4796d93dfab2d9a605 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d62f069c4c0e4a4796d93dfab2d9a605 2024-12-02T22:13:56,751 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T22:13:56,751 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T22:13:56,752 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into d62f069c4c0e4a4796d93dfab2d9a605(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:56,752 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:56,752 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177636527; duration=0sec 2024-12-02T22:13:56,752 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:56,752 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:13:56,753 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T22:13:56,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T22:13:56,754 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T22:13:56,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T22:13:56,755 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T22:13:56,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T22:13:56,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177696803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177696804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177696804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177696804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177697025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177697026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177697026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177697026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,042 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/d303102ca3a548fbb591dbbe0edea178 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d303102ca3a548fbb591dbbe0edea178 2024-12-02T22:13:57,060 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into d303102ca3a548fbb591dbbe0edea178(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:57,060 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:57,060 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177636527; duration=0sec 2024-12-02T22:13:57,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:57,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:13:57,122 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/0eb2706fe47b43a698413b8f9608ec4e 2024-12-02T22:13:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/fa969eefb40e437bb413bacbfa3067e1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1 2024-12-02T22:13:57,151 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1, entries=150, sequenceid=149, filesize=11.9 K 2024-12-02T22:13:57,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/76fc0e5972ed4b578373a499ee3af6c1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1 2024-12-02T22:13:57,162 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1, entries=150, sequenceid=149, filesize=11.9 K 2024-12-02T22:13:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/0eb2706fe47b43a698413b8f9608ec4e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e 2024-12-02T22:13:57,174 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e, entries=150, sequenceid=149, filesize=11.9 K 2024-12-02T22:13:57,180 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~122.60 KB/125545, heapSize ~349.26 KB/357640, currentSize=67.09 KB/68700 for 8de9171fd3e9b964cfb173aae742ae78 in 626ms, sequenceid=149, compaction requested=false 2024-12-02T22:13:57,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:57,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T22:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T22:13:57,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T22:13:57,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-12-02T22:13:57,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1260 sec 2024-12-02T22:13:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T22:13:57,204 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:13:57,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:13:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T22:13:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:57,209 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:13:57,210 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:13:57,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:13:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:13:57,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:57,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:57,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e3bfc8f847c44fceadccea8f176a461b is 50, key is test_row_0/A:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:57,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T22:13:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177697385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177697382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177697396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177697400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741861_1037 (size=12151) 2024-12-02T22:13:57,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177697503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177697513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177697514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177697515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T22:13:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T22:13:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177697724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177697724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177697724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:57,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177697725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:57,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e3bfc8f847c44fceadccea8f176a461b 2024-12-02T22:13:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:57,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3dd032787ad64177858ad954f6c77f29 is 50, key is test_row_0/B:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:57,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T22:13:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:57,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741862_1038 (size=12151) 2024-12-02T22:13:57,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3dd032787ad64177858ad954f6c77f29 2024-12-02T22:13:57,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/db4086d7fbb44e0783d1c5bbfc2728e7 is 50, key is test_row_0/C:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:57,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741863_1039 (size=12151) 2024-12-02T22:13:57,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/db4086d7fbb44e0783d1c5bbfc2728e7 2024-12-02T22:13:57,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e3bfc8f847c44fceadccea8f176a461b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b 2024-12-02T22:13:57,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T22:13:57,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3dd032787ad64177858ad954f6c77f29 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29 2024-12-02T22:13:57,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T22:13:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/db4086d7fbb44e0783d1c5bbfc2728e7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7 2024-12-02T22:13:57,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T22:13:57,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8de9171fd3e9b964cfb173aae742ae78 in 602ms, sequenceid=166, compaction requested=true 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:13:57,950 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:57,950 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:13:57,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:57,952 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:57,952 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:13:57,952 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,952 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:57,952 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f6fe5b934a6d4dea81b053acd884e923, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.7 K 2024-12-02T22:13:57,952 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:13:57,952 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:57,953 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d303102ca3a548fbb591dbbe0edea178, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.7 K 2024-12-02T22:13:57,953 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6fe5b934a6d4dea81b053acd884e923, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:57,953 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d303102ca3a548fbb591dbbe0edea178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:57,954 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa969eefb40e437bb413bacbfa3067e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733177635978 2024-12-02T22:13:57,954 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 76fc0e5972ed4b578373a499ee3af6c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733177635978 2024-12-02T22:13:57,955 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bfc8f847c44fceadccea8f176a461b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:57,955 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dd032787ad64177858ad954f6c77f29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:57,982 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:57,983 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/058592f596874187be5ee5d624fc7150 is 50, key is test_row_0/B:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:57,987 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#28 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:57,988 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/4a360754d110494faf9402dd222c0476 is 50, key is test_row_0/A:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:58,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T22:13:58,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:58,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:13:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:58,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741865_1041 (size=12459) 2024-12-02T22:13:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741864_1040 (size=12459) 2024-12-02T22:13:58,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/2a30dcf5663d407c8a17cccf755dea73 is 50, key is test_row_0/A:col10/1733177637382/Put/seqid=0 2024-12-02T22:13:58,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:58,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:58,051 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/058592f596874187be5ee5d624fc7150 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/058592f596874187be5ee5d624fc7150 2024-12-02T22:13:58,060 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 058592f596874187be5ee5d624fc7150(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:58,060 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:58,060 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177637950; duration=0sec 2024-12-02T22:13:58,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:58,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:13:58,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:58,067 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:58,067 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:13:58,067 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:58,067 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d62f069c4c0e4a4796d93dfab2d9a605, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.7 K 2024-12-02T22:13:58,068 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d62f069c4c0e4a4796d93dfab2d9a605, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177635802 2024-12-02T22:13:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,068 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eb2706fe47b43a698413b8f9608ec4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733177635978 2024-12-02T22:13:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177698055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,069 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting db4086d7fbb44e0783d1c5bbfc2728e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177698057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177698066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177698069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,088 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:58,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741866_1042 (size=12151) 2024-12-02T22:13:58,089 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/aa9c1b746b7346feab29c8baeec33ab1 is 50, key is test_row_0/C:col10/1733177637343/Put/seqid=0 2024-12-02T22:13:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741867_1043 (size=12459) 2024-12-02T22:13:58,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177698174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177698174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177698183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177698184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:58,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177698393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177698394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177698394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177698403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,423 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/4a360754d110494faf9402dd222c0476 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/4a360754d110494faf9402dd222c0476 2024-12-02T22:13:58,432 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into 4a360754d110494faf9402dd222c0476(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:58,432 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:58,432 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177637950; duration=0sec 2024-12-02T22:13:58,433 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:58,433 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:13:58,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/2a30dcf5663d407c8a17cccf755dea73 2024-12-02T22:13:58,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/854fe06294474bbda312533d5fc9c330 is 50, key is test_row_0/B:col10/1733177637382/Put/seqid=0 2024-12-02T22:13:58,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741868_1044 (size=12151) 2024-12-02T22:13:58,550 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/aa9c1b746b7346feab29c8baeec33ab1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/aa9c1b746b7346feab29c8baeec33ab1 2024-12-02T22:13:58,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/854fe06294474bbda312533d5fc9c330 2024-12-02T22:13:58,564 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into aa9c1b746b7346feab29c8baeec33ab1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:58,564 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:58,565 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177637950; duration=0sec 2024-12-02T22:13:58,565 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:58,565 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:13:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/9aacdc50afa0447c86635e305d6a1ec0 is 50, key is test_row_0/C:col10/1733177637382/Put/seqid=0 2024-12-02T22:13:58,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741869_1045 (size=12151) 2024-12-02T22:13:58,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/9aacdc50afa0447c86635e305d6a1ec0 2024-12-02T22:13:58,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/2a30dcf5663d407c8a17cccf755dea73 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73 2024-12-02T22:13:58,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73, entries=150, sequenceid=184, filesize=11.9 K 2024-12-02T22:13:58,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/854fe06294474bbda312533d5fc9c330 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330 2024-12-02T22:13:58,687 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330, entries=150, sequenceid=184, filesize=11.9 K 2024-12-02T22:13:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/9aacdc50afa0447c86635e305d6a1ec0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0 2024-12-02T22:13:58,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177698704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177698713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177698714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,716 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0, entries=150, sequenceid=184, filesize=11.9 K 2024-12-02T22:13:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177698714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:58,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8de9171fd3e9b964cfb173aae742ae78 in 729ms, sequenceid=184, compaction requested=false 2024-12-02T22:13:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T22:13:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T22:13:58,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T22:13:58,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-12-02T22:13:58,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5420 sec 2024-12-02T22:13:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:59,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:59,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b114bd26a51c4d68a87ef8b66212f5df is 50, key is test_row_0/A:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741870_1046 (size=14541) 2024-12-02T22:13:59,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177699267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177699274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177699275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177699287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T22:13:59,364 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:13:59,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:13:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T22:13:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T22:13:59,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:13:59,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:13:59,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:13:59,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177699383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177699393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177699393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177699404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T22:13:59,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T22:13:59,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:59,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177699593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177699603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177699604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177699613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T22:13:59,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:59,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b114bd26a51c4d68a87ef8b66212f5df 2024-12-02T22:13:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:13:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T22:13:59,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6fad2752cdb64947ac073733ab2acfa0 is 50, key is test_row_0/B:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741871_1047 (size=12151) 2024-12-02T22:13:59,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6fad2752cdb64947ac073733ab2acfa0 2024-12-02T22:13:59,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/debded6d84b547019ed3e3c6ef630f86 is 50, key is test_row_0/C:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741872_1048 (size=12151) 2024-12-02T22:13:59,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/debded6d84b547019ed3e3c6ef630f86 2024-12-02T22:13:59,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b114bd26a51c4d68a87ef8b66212f5df as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df 2024-12-02T22:13:59,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df, entries=200, sequenceid=202, filesize=14.2 K 2024-12-02T22:13:59,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/6fad2752cdb64947ac073733ab2acfa0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0 2024-12-02T22:13:59,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0, entries=150, sequenceid=202, filesize=11.9 K 2024-12-02T22:13:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/debded6d84b547019ed3e3c6ef630f86 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86 2024-12-02T22:13:59,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86, entries=150, sequenceid=202, filesize=11.9 K 2024-12-02T22:13:59,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8de9171fd3e9b964cfb173aae742ae78 in 584ms, sequenceid=202, compaction requested=true 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:13:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:13:59,800 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:59,801 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:59,803 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:59,803 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:59,803 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:13:59,803 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:13:59,803 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,803 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,803 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/4a360754d110494faf9402dd222c0476, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=38.2 K 2024-12-02T22:13:59,803 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/aa9c1b746b7346feab29c8baeec33ab1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.9 K 2024-12-02T22:13:59,804 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9c1b746b7346feab29c8baeec33ab1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:59,804 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a360754d110494faf9402dd222c0476, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:59,805 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a30dcf5663d407c8a17cccf755dea73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177637377 2024-12-02T22:13:59,805 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aacdc50afa0447c86635e305d6a1ec0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177637377 2024-12-02T22:13:59,805 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b114bd26a51c4d68a87ef8b66212f5df, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638052 2024-12-02T22:13:59,805 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting debded6d84b547019ed3e3c6ef630f86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638063 2024-12-02T22:13:59,830 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:59,831 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/e3ad28f1c9b04946ab129cd54270770e is 50, key is test_row_0/C:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T22:13:59,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:13:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:13:59,836 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:59,837 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b2e6acab364044e2b41193f2d84a914c is 50, key is test_row_0/A:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/6c37254e6a9b4735869d7881890ad4f2 is 50, key is test_row_0/A:col10/1733177639268/Put/seqid=0 2024-12-02T22:13:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741874_1050 (size=12561) 2024-12-02T22:13:59,903 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/e3ad28f1c9b04946ab129cd54270770e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e3ad28f1c9b04946ab129cd54270770e 2024-12-02T22:13:59,912 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into e3ad28f1c9b04946ab129cd54270770e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:59,912 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:59,912 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177639800; duration=0sec 2024-12-02T22:13:59,912 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:13:59,912 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:13:59,912 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:13:59,916 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:13:59,916 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:13:59,916 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:13:59,916 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/058592f596874187be5ee5d624fc7150, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=35.9 K 2024-12-02T22:13:59,917 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058592f596874187be5ee5d624fc7150, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177636672 2024-12-02T22:13:59,918 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 854fe06294474bbda312533d5fc9c330, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177637377 2024-12-02T22:13:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:13:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:13:59,919 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fad2752cdb64947ac073733ab2acfa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638063 2024-12-02T22:13:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741873_1049 (size=12561) 2024-12-02T22:13:59,931 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b2e6acab364044e2b41193f2d84a914c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b2e6acab364044e2b41193f2d84a914c 2024-12-02T22:13:59,939 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into b2e6acab364044e2b41193f2d84a914c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:13:59,939 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:13:59,939 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177639800; duration=0sec 2024-12-02T22:13:59,939 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:13:59,939 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:13:59,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741875_1051 (size=12151) 2024-12-02T22:13:59,964 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:13:59,965 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/176a2eafd0f6456680c4ae183256c6c4 is 50, key is test_row_0/B:col10/1733177639214/Put/seqid=0 2024-12-02T22:13:59,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177699962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177699967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177699967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:13:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177699974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:13:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:13:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:13:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T22:14:00,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741876_1052 (size=12561) 2024-12-02T22:14:00,020 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/176a2eafd0f6456680c4ae183256c6c4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/176a2eafd0f6456680c4ae183256c6c4 2024-12-02T22:14:00,032 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 176a2eafd0f6456680c4ae183256c6c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:00,032 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:00,032 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177639800; duration=0sec 2024-12-02T22:14:00,033 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:00,033 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:00,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177700084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177700084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177700084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177700084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177700304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177700304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177700304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177700305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/6c37254e6a9b4735869d7881890ad4f2 2024-12-02T22:14:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/54fb87088ec2495580d670ebb3196cf6 is 50, key is test_row_0/B:col10/1733177639268/Put/seqid=0 2024-12-02T22:14:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741877_1053 (size=12151) 2024-12-02T22:14:00,392 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/54fb87088ec2495580d670ebb3196cf6 2024-12-02T22:14:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ae256155f1d045c092b6c321e51565e4 is 50, key is test_row_0/C:col10/1733177639268/Put/seqid=0 2024-12-02T22:14:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741878_1054 (size=12151) 2024-12-02T22:14:00,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ae256155f1d045c092b6c321e51565e4 2024-12-02T22:14:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/6c37254e6a9b4735869d7881890ad4f2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2 2024-12-02T22:14:00,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2, entries=150, sequenceid=220, filesize=11.9 K 2024-12-02T22:14:00,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/54fb87088ec2495580d670ebb3196cf6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6 2024-12-02T22:14:00,458 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6, entries=150, sequenceid=220, filesize=11.9 K 2024-12-02T22:14:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ae256155f1d045c092b6c321e51565e4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4 2024-12-02T22:14:00,471 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4, entries=150, sequenceid=220, filesize=11.9 K 2024-12-02T22:14:00,473 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8de9171fd3e9b964cfb173aae742ae78 in 640ms, sequenceid=220, compaction requested=false 2024-12-02T22:14:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T22:14:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T22:14:00,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T22:14:00,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1070 sec 2024-12-02T22:14:00,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1190 sec 2024-12-02T22:14:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T22:14:00,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:00,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T22:14:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:00,512 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:00,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:00,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:00,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:14:00,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:00,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:00,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/579f1c1f38d940cba7ddc1f45d779a49 is 50, key is test_row_0/A:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:00,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741879_1055 (size=12147) 2024-12-02T22:14:00,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/579f1c1f38d940cba7ddc1f45d779a49 2024-12-02T22:14:00,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/7354a9d74b764a0fbe1aaca071b60b82 is 50, key is test_row_0/B:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741880_1056 (size=9757) 2024-12-02T22:14:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177700621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177700623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177700625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177700631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177700632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:00,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177700733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177700743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177700743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177700743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177700743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:00,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177700943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177700954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177700954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177700955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177700955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:00,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:00,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/7354a9d74b764a0fbe1aaca071b60b82 2024-12-02T22:14:01,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/3e187f7e98b44030b3f705e135fdb28b is 50, key is test_row_0/C:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:01,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741881_1057 (size=9757) 2024-12-02T22:14:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:01,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177701253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177701264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177701264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177701264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177701264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:01,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/3e187f7e98b44030b3f705e135fdb28b 2024-12-02T22:14:01,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/579f1c1f38d940cba7ddc1f45d779a49 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49 2024-12-02T22:14:01,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49, entries=150, sequenceid=238, filesize=11.9 K 2024-12-02T22:14:01,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/7354a9d74b764a0fbe1aaca071b60b82 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82 2024-12-02T22:14:01,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82, entries=100, sequenceid=238, filesize=9.5 K 2024-12-02T22:14:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/3e187f7e98b44030b3f705e135fdb28b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b 2024-12-02T22:14:01,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b, entries=100, sequenceid=238, filesize=9.5 K 2024-12-02T22:14:01,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8de9171fd3e9b964cfb173aae742ae78 in 1040ms, sequenceid=238, compaction requested=true 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:01,583 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:01,583 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:01,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:01,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:01,585 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:01,585 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:01,585 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,585 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b2e6acab364044e2b41193f2d84a914c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.0 K 2024-12-02T22:14:01,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:01,586 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e6acab364044e2b41193f2d84a914c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638063 2024-12-02T22:14:01,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:01,586 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,587 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/176a2eafd0f6456680c4ae183256c6c4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=33.7 K 2024-12-02T22:14:01,587 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c37254e6a9b4735869d7881890ad4f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733177639253 2024-12-02T22:14:01,587 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 176a2eafd0f6456680c4ae183256c6c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638063 2024-12-02T22:14:01,588 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579f1c1f38d940cba7ddc1f45d779a49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639943 2024-12-02T22:14:01,588 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fb87088ec2495580d670ebb3196cf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733177639253 2024-12-02T22:14:01,589 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7354a9d74b764a0fbe1aaca071b60b82, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639966 2024-12-02T22:14:01,602 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:01,603 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/22a1bfd218f84f50b0cbfed12b70ea3d is 50, key is test_row_0/B:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:01,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T22:14:01,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:01,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:01,609 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:01,609 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/3828f294dbf54e6cb148e08fb7ffcfc7 is 50, key is test_row_0/A:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:01,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/982da7575b2740d393e6a82e3c0f5bd7 is 50, key is test_row_0/A:col10/1733177640628/Put/seqid=0 2024-12-02T22:14:01,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741883_1059 (size=12663) 2024-12-02T22:14:01,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741884_1060 (size=12151) 2024-12-02T22:14:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741882_1058 (size=12663) 2024-12-02T22:14:01,630 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/22a1bfd218f84f50b0cbfed12b70ea3d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/22a1bfd218f84f50b0cbfed12b70ea3d 2024-12-02T22:14:01,641 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 22a1bfd218f84f50b0cbfed12b70ea3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:01,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:01,641 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177641583; duration=0sec 2024-12-02T22:14:01,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:01,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:01,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:01,642 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:01,642 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:01,643 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:01,643 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e3ad28f1c9b04946ab129cd54270770e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=33.7 K 2024-12-02T22:14:01,643 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ad28f1c9b04946ab129cd54270770e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733177638063 2024-12-02T22:14:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:01,646 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ae256155f1d045c092b6c321e51565e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733177639253 2024-12-02T22:14:01,648 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e187f7e98b44030b3f705e135fdb28b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639966 2024-12-02T22:14:01,670 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:01,671 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/909e8185adc546b38eee0e1e141cdc99 is 50, key is test_row_0/C:col10/1733177639966/Put/seqid=0 2024-12-02T22:14:01,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741885_1061 (size=12663) 2024-12-02T22:14:01,701 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/909e8185adc546b38eee0e1e141cdc99 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/909e8185adc546b38eee0e1e141cdc99 2024-12-02T22:14:01,710 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into 909e8185adc546b38eee0e1e141cdc99(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:01,711 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:01,711 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177641583; duration=0sec 2024-12-02T22:14:01,711 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:01,711 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:01,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:01,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177701786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177701794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177701798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177701799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177701805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177701913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177701914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177701914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177701914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177701914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/982da7575b2740d393e6a82e3c0f5bd7 2024-12-02T22:14:02,046 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/3828f294dbf54e6cb148e08fb7ffcfc7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/3828f294dbf54e6cb148e08fb7ffcfc7 2024-12-02T22:14:02,056 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into 3828f294dbf54e6cb148e08fb7ffcfc7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:02,056 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:02,056 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177641583; duration=0sec 2024-12-02T22:14:02,057 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:02,057 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:02,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/9e7dbee483dc4f16bc6a4367346aa350 is 50, key is test_row_0/B:col10/1733177640628/Put/seqid=0 2024-12-02T22:14:02,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741886_1062 (size=12151) 2024-12-02T22:14:02,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177702123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177702124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177702123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177702133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177702133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177702433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177702433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177702443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177702443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177702444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/9e7dbee483dc4f16bc6a4367346aa350 2024-12-02T22:14:02,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/6ebe5ebb289d48c8a3fae27ab347eb67 is 50, key is test_row_0/C:col10/1733177640628/Put/seqid=0 2024-12-02T22:14:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741887_1063 (size=12151) 2024-12-02T22:14:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:02,917 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/6ebe5ebb289d48c8a3fae27ab347eb67 2024-12-02T22:14:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/982da7575b2740d393e6a82e3c0f5bd7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7 2024-12-02T22:14:02,931 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7, entries=150, sequenceid=256, filesize=11.9 K 2024-12-02T22:14:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/9e7dbee483dc4f16bc6a4367346aa350 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350 2024-12-02T22:14:02,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350, entries=150, sequenceid=256, filesize=11.9 K 2024-12-02T22:14:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/6ebe5ebb289d48c8a3fae27ab347eb67 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67 2024-12-02T22:14:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177702943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177702953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67, entries=150, sequenceid=256, filesize=11.9 K 2024-12-02T22:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:02,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177702953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:02,958 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8de9171fd3e9b964cfb173aae742ae78 in 1351ms, sequenceid=256, compaction requested=false 2024-12-02T22:14:02,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:02,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:02,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T22:14:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T22:14:02,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T22:14:02,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4450 sec 2024-12-02T22:14:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:02,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:02,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4620 sec 2024-12-02T22:14:02,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/08fbf3e0ffc74f5cbd578302aa875a33 is 50, key is test_row_0/A:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:03,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741888_1064 (size=14741) 2024-12-02T22:14:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177703037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177703039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177703153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177703154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177703363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177703364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/08fbf3e0ffc74f5cbd578302aa875a33 2024-12-02T22:14:03,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/256463f823fc4ddb84f3687db027af73 is 50, key is test_row_0/B:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:03,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741889_1065 (size=12301) 2024-12-02T22:14:03,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/256463f823fc4ddb84f3687db027af73 2024-12-02T22:14:03,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/a9b5fdaff91841d59994072926e69a50 is 50, key is test_row_0/C:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:03,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741890_1066 (size=12301) 2024-12-02T22:14:03,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/a9b5fdaff91841d59994072926e69a50 2024-12-02T22:14:03,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/08fbf3e0ffc74f5cbd578302aa875a33 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33 2024-12-02T22:14:03,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33, entries=200, sequenceid=275, filesize=14.4 K 2024-12-02T22:14:03,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/256463f823fc4ddb84f3687db027af73 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73 2024-12-02T22:14:03,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T22:14:03,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/a9b5fdaff91841d59994072926e69a50 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50 2024-12-02T22:14:03,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T22:14:03,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8de9171fd3e9b964cfb173aae742ae78 in 537ms, sequenceid=275, compaction requested=true 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:03,505 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:03,505 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:03,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:03,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:03,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:03,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:03,506 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:03,506 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/3828f294dbf54e6cb148e08fb7ffcfc7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=38.6 K 2024-12-02T22:14:03,507 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:03,507 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3828f294dbf54e6cb148e08fb7ffcfc7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639269 2024-12-02T22:14:03,507 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:03,507 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:03,508 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/22a1bfd218f84f50b0cbfed12b70ea3d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.2 K 2024-12-02T22:14:03,508 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 982da7575b2740d393e6a82e3c0f5bd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733177640618 2024-12-02T22:14:03,509 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a1bfd218f84f50b0cbfed12b70ea3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639269 2024-12-02T22:14:03,509 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08fbf3e0ffc74f5cbd578302aa875a33, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641784 2024-12-02T22:14:03,509 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7dbee483dc4f16bc6a4367346aa350, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733177640618 2024-12-02T22:14:03,510 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 256463f823fc4ddb84f3687db027af73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641794 2024-12-02T22:14:03,522 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:03,523 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/831d9735c1904214bb183c96389c0d1f is 50, key is test_row_0/B:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:03,535 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:03,536 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/feb0d0f68e1c4b1daa611b0c48b34df6 is 50, key is test_row_0/A:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:03,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741891_1067 (size=12915) 2024-12-02T22:14:03,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741892_1068 (size=12915) 2024-12-02T22:14:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:03,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:03,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:03,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:03,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/97e736e383d94ff398c52883898b421e is 50, key is test_row_0/A:col10/1733177643674/Put/seqid=0 2024-12-02T22:14:03,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741893_1069 (size=14741) 2024-12-02T22:14:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177703708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177703710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177703823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177703823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177703953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,957 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,969 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/831d9735c1904214bb183c96389c0d1f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/831d9735c1904214bb183c96389c0d1f 2024-12-02T22:14:03,971 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/feb0d0f68e1c4b1daa611b0c48b34df6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/feb0d0f68e1c4b1daa611b0c48b34df6 2024-12-02T22:14:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177703973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177703974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:03,978 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,979 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:03,987 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 831d9735c1904214bb183c96389c0d1f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:03,987 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:03,987 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177643505; duration=0sec 2024-12-02T22:14:03,988 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:03,988 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:03,988 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:03,989 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into feb0d0f68e1c4b1daa611b0c48b34df6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:03,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:03,989 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177643505; duration=0sec 2024-12-02T22:14:03,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:03,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:03,992 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:03,992 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:03,992 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:03,992 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/909e8185adc546b38eee0e1e141cdc99, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.2 K 2024-12-02T22:14:03,993 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 909e8185adc546b38eee0e1e141cdc99, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733177639269 2024-12-02T22:14:03,993 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ebe5ebb289d48c8a3fae27ab347eb67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733177640618 2024-12-02T22:14:03,994 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b5fdaff91841d59994072926e69a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641794 2024-12-02T22:14:04,007 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:04,007 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/906740eb10ad48dcb216f32425ac9c1f is 50, key is test_row_0/C:col10/1733177641796/Put/seqid=0 2024-12-02T22:14:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741894_1070 (size=12915) 2024-12-02T22:14:04,023 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/906740eb10ad48dcb216f32425ac9c1f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/906740eb10ad48dcb216f32425ac9c1f 2024-12-02T22:14:04,030 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into 906740eb10ad48dcb216f32425ac9c1f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:04,030 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:04,030 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177643505; duration=0sec 2024-12-02T22:14:04,030 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:04,030 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177704033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177704033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/97e736e383d94ff398c52883898b421e 2024-12-02T22:14:04,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8628ca5a2f35448191783692e8223d88 is 50, key is test_row_0/B:col10/1733177643674/Put/seqid=0 2024-12-02T22:14:04,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741895_1071 (size=12301) 2024-12-02T22:14:04,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8628ca5a2f35448191783692e8223d88 2024-12-02T22:14:04,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/4f492cb7feb049388dc3a0f29775dec5 is 50, key is test_row_0/C:col10/1733177643674/Put/seqid=0 2024-12-02T22:14:04,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741896_1072 (size=12301) 2024-12-02T22:14:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177704343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177704344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/4f492cb7feb049388dc3a0f29775dec5 2024-12-02T22:14:04,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/97e736e383d94ff398c52883898b421e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e 2024-12-02T22:14:04,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e, entries=200, sequenceid=294, filesize=14.4 K 2024-12-02T22:14:04,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8628ca5a2f35448191783692e8223d88 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88 2024-12-02T22:14:04,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T22:14:04,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/4f492cb7feb049388dc3a0f29775dec5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5 2024-12-02T22:14:04,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T22:14:04,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8de9171fd3e9b964cfb173aae742ae78 in 899ms, sequenceid=294, compaction requested=false 2024-12-02T22:14:04,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T22:14:04,673 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:04,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T22:14:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T22:14:04,677 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:04,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:04,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T22:14:04,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T22:14:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:04,833 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:14:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/5b4e9d94438844dab565b7b5a79b82a1 is 50, key is test_row_0/A:col10/1733177643706/Put/seqid=0 2024-12-02T22:14:04,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741897_1073 (size=12301) 2024-12-02T22:14:04,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/5b4e9d94438844dab565b7b5a79b82a1 2024-12-02T22:14:04,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3f99da527edf451583a0e9ee43bb9c92 is 50, key is test_row_0/B:col10/1733177643706/Put/seqid=0 2024-12-02T22:14:04,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741898_1074 (size=12301) 2024-12-02T22:14:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177704916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177704920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T22:14:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177705023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177705033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177705233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177705243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,280 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3f99da527edf451583a0e9ee43bb9c92 2024-12-02T22:14:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ef72f422e97945539a5f568e3d45bc06 is 50, key is test_row_0/C:col10/1733177643706/Put/seqid=0 2024-12-02T22:14:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T22:14:05,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741899_1075 (size=12301) 2024-12-02T22:14:05,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177705543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177705554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:05,712 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ef72f422e97945539a5f568e3d45bc06 2024-12-02T22:14:05,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/5b4e9d94438844dab565b7b5a79b82a1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1 2024-12-02T22:14:05,728 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1, entries=150, sequenceid=310, filesize=12.0 K 2024-12-02T22:14:05,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/3f99da527edf451583a0e9ee43bb9c92 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92 2024-12-02T22:14:05,736 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92, entries=150, sequenceid=310, filesize=12.0 K 2024-12-02T22:14:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ef72f422e97945539a5f568e3d45bc06 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06 2024-12-02T22:14:05,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06, entries=150, sequenceid=310, filesize=12.0 K 2024-12-02T22:14:05,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8de9171fd3e9b964cfb173aae742ae78 in 914ms, sequenceid=310, compaction requested=true 2024-12-02T22:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:05,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T22:14:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T22:14:05,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T22:14:05,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-12-02T22:14:05,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0820 sec 2024-12-02T22:14:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T22:14:05,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:05,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T22:14:05,821 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T22:14:05,822 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:05,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T22:14:05,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T22:14:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:05,980 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:05,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/715709401bff46a1ad99507c431b640d is 50, key is test_row_0/A:col10/1733177644916/Put/seqid=0 2024-12-02T22:14:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177706025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741900_1076 (size=12301) 2024-12-02T22:14:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/715709401bff46a1ad99507c431b640d 2024-12-02T22:14:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177706035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177706050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/ad38ba254cff4afba4b484a087dde5b9 is 50, key is test_row_0/B:col10/1733177644916/Put/seqid=0 2024-12-02T22:14:06,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177706055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177706063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741901_1077 (size=12301) 2024-12-02T22:14:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T22:14:06,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177706144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177706154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177706163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177706354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177706363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177706373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T22:14:06,495 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/ad38ba254cff4afba4b484a087dde5b9 2024-12-02T22:14:06,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/c3e583af21744a5c9779b0ef8f538aa2 is 50, key is test_row_0/C:col10/1733177644916/Put/seqid=0 2024-12-02T22:14:06,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741902_1078 (size=12301) 2024-12-02T22:14:06,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/c3e583af21744a5c9779b0ef8f538aa2 2024-12-02T22:14:06,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/715709401bff46a1ad99507c431b640d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d 2024-12-02T22:14:06,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d, entries=150, sequenceid=329, filesize=12.0 K 2024-12-02T22:14:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/ad38ba254cff4afba4b484a087dde5b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9 2024-12-02T22:14:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9, entries=150, sequenceid=329, filesize=12.0 K 2024-12-02T22:14:06,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/c3e583af21744a5c9779b0ef8f538aa2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2 2024-12-02T22:14:06,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2, entries=150, sequenceid=329, filesize=12.0 K 2024-12-02T22:14:06,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8de9171fd3e9b964cfb173aae742ae78 in 562ms, sequenceid=329, compaction requested=true 2024-12-02T22:14:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T22:14:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T22:14:06,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T22:14:06,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-12-02T22:14:06,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 733 msec 2024-12-02T22:14:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:06,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:14:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:06,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a6da67afb2cb460bb9db74ac651be847 is 50, key is test_row_0/A:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:06,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741903_1079 (size=14741) 2024-12-02T22:14:06,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a6da67afb2cb460bb9db74ac651be847 2024-12-02T22:14:06,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e495e3a7fe7741ceac0433ce62c3c4e2 is 50, key is test_row_0/B:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741904_1080 (size=12301) 2024-12-02T22:14:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177706718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177706724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177706727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177706833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177706833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177706843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T22:14:06,964 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:06,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T22:14:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T22:14:06,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:06,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:06,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177707043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177707043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177707053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177707064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,067 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T22:14:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177707073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,078 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e495e3a7fe7741ceac0433ce62c3c4e2 2024-12-02T22:14:07,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T22:14:07,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/60da36b2551a478b8a834fd25e77510a is 50, key is test_row_0/C:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741905_1081 (size=12301) 2024-12-02T22:14:07,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T22:14:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T22:14:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177707354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177707354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177707373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:07,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:07,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T22:14:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:07,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/60da36b2551a478b8a834fd25e77510a 2024-12-02T22:14:07,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a6da67afb2cb460bb9db74ac651be847 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847 2024-12-02T22:14:07,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847, entries=200, sequenceid=344, filesize=14.4 K 2024-12-02T22:14:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e495e3a7fe7741ceac0433ce62c3c4e2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2 2024-12-02T22:14:07,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2, entries=150, sequenceid=344, filesize=12.0 K 2024-12-02T22:14:07,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/60da36b2551a478b8a834fd25e77510a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a 2024-12-02T22:14:07,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a, entries=150, sequenceid=344, filesize=12.0 K 2024-12-02T22:14:07,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8de9171fd3e9b964cfb173aae742ae78 in 922ms, sequenceid=344, compaction requested=true 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:07,591 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:07,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:07,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T22:14:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,592 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:14:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T22:14:07,594 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66999 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:14:07,595 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:07,595 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,595 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62119 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:14:07,595 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/feb0d0f68e1c4b1daa611b0c48b34df6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=65.4 K 2024-12-02T22:14:07,595 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:07,595 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,595 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/831d9735c1904214bb183c96389c0d1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=60.7 K 2024-12-02T22:14:07,595 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb0d0f68e1c4b1daa611b0c48b34df6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641794 2024-12-02T22:14:07,596 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 831d9735c1904214bb183c96389c0d1f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641794 2024-12-02T22:14:07,596 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e736e383d94ff398c52883898b421e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177643016 2024-12-02T22:14:07,597 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8628ca5a2f35448191783692e8223d88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177643035 2024-12-02T22:14:07,597 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b4e9d94438844dab565b7b5a79b82a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733177643693 2024-12-02T22:14:07,598 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f99da527edf451583a0e9ee43bb9c92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733177643693 2024-12-02T22:14:07,598 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 715709401bff46a1ad99507c431b640d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733177644907 2024-12-02T22:14:07,598 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ad38ba254cff4afba4b484a087dde5b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733177644907 2024-12-02T22:14:07,599 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6da67afb2cb460bb9db74ac651be847, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646022 2024-12-02T22:14:07,599 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e495e3a7fe7741ceac0433ce62c3c4e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646032 2024-12-02T22:14:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b9406390b34f4350ad61309caca77820 is 50, key is test_row_0/A:col10/1733177646721/Put/seqid=0 2024-12-02T22:14:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741906_1082 (size=12301) 2024-12-02T22:14:07,630 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b9406390b34f4350ad61309caca77820 2024-12-02T22:14:07,643 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#70 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:07,644 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/583cff4e216843698b4f433afcd42ab9 is 50, key is test_row_0/A:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:07,645 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:07,646 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/1376b8d434e84898b68e6af778d6895c is 50, key is test_row_0/B:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/0c1017c3bd644eba9bd2aeca4ee44e9e is 50, key is test_row_0/B:col10/1733177646721/Put/seqid=0 2024-12-02T22:14:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741907_1083 (size=13085) 2024-12-02T22:14:07,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741909_1085 (size=12301) 2024-12-02T22:14:07,664 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/0c1017c3bd644eba9bd2aeca4ee44e9e 2024-12-02T22:14:07,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/f2c7d21c64744b75a295dd896b75b1ff is 50, key is test_row_0/C:col10/1733177646721/Put/seqid=0 2024-12-02T22:14:07,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741908_1084 (size=13085) 2024-12-02T22:14:07,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741910_1086 (size=12301) 2024-12-02T22:14:07,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/f2c7d21c64744b75a295dd896b75b1ff 2024-12-02T22:14:07,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/b9406390b34f4350ad61309caca77820 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820 2024-12-02T22:14:07,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/1376b8d434e84898b68e6af778d6895c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/1376b8d434e84898b68e6af778d6895c 2024-12-02T22:14:07,713 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 1376b8d434e84898b68e6af778d6895c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:07,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:07,713 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=11, startTime=1733177647591; duration=0sec 2024-12-02T22:14:07,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:07,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:07,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:14:07,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820, entries=150, sequenceid=361, filesize=12.0 K 2024-12-02T22:14:07,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/0c1017c3bd644eba9bd2aeca4ee44e9e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e 2024-12-02T22:14:07,724 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62119 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:14:07,724 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:07,724 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,724 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/906740eb10ad48dcb216f32425ac9c1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=60.7 K 2024-12-02T22:14:07,725 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 906740eb10ad48dcb216f32425ac9c1f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733177641794 2024-12-02T22:14:07,725 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f492cb7feb049388dc3a0f29775dec5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177643035 2024-12-02T22:14:07,726 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ef72f422e97945539a5f568e3d45bc06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733177643693 2024-12-02T22:14:07,727 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e583af21744a5c9779b0ef8f538aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733177644907 2024-12-02T22:14:07,727 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 60da36b2551a478b8a834fd25e77510a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646032 2024-12-02T22:14:07,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e, entries=150, sequenceid=361, filesize=12.0 K 2024-12-02T22:14:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/f2c7d21c64744b75a295dd896b75b1ff as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff 2024-12-02T22:14:07,742 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff, entries=150, sequenceid=361, filesize=12.0 K 2024-12-02T22:14:07,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 8de9171fd3e9b964cfb173aae742ae78 in 151ms, sequenceid=361, compaction requested=false 2024-12-02T22:14:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T22:14:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T22:14:07,750 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:07,751 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/9ec4d11cb3954d8e8a31575e5039c717 is 50, key is test_row_0/C:col10/1733177646032/Put/seqid=0 2024-12-02T22:14:07,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T22:14:07,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 787 msec 2024-12-02T22:14:07,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 776 msec 2024-12-02T22:14:07,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741911_1087 (size=13085) 2024-12-02T22:14:07,790 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/9ec4d11cb3954d8e8a31575e5039c717 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9ec4d11cb3954d8e8a31575e5039c717 2024-12-02T22:14:07,800 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into 9ec4d11cb3954d8e8a31575e5039c717(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:07,800 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:07,800 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=11, startTime=1733177647591; duration=0sec 2024-12-02T22:14:07,801 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:07,801 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:07,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:07,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f02724b6b2964e61a53cee971a24d592 is 50, key is test_row_0/A:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:07,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741912_1088 (size=12301) 2024-12-02T22:14:07,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f02724b6b2964e61a53cee971a24d592 2024-12-02T22:14:07,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8981900ebdc44daf8b98011da13fec3a is 50, key is test_row_0/B:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:07,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741913_1089 (size=12301) 2024-12-02T22:14:07,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8981900ebdc44daf8b98011da13fec3a 2024-12-02T22:14:07,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177647954, max timestamp=1733177647992Num uniques 859; 2024-12-02T22:14:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177647954, max timestamp=1733177647992Num uniques 859; 2024-12-02T22:14:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177647954, max timestamp=1733177647992Num uniques 859; 2024-12-02T22:14:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:08,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/d7c74966e29c435b805ada5381413dda is 50, key is test_row_0/C:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:08,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741914_1090 (size=12301) 2024-12-02T22:14:08,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177708053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,064 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/583cff4e216843698b4f433afcd42ab9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/583cff4e216843698b4f433afcd42ab9 2024-12-02T22:14:08,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177708056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177708058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,072 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into 583cff4e216843698b4f433afcd42ab9(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:08,072 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:08,072 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=11, startTime=1733177647591; duration=0sec 2024-12-02T22:14:08,073 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:08,073 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T22:14:08,105 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:08,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T22:14:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T22:14:08,110 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:08,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:08,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:08,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177708163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177708174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177708174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T22:14:08,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T22:14:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177708384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177708393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177708393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/d7c74966e29c435b805ada5381413dda 2024-12-02T22:14:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T22:14:08,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T22:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:08,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f02724b6b2964e61a53cee971a24d592 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592 2024-12-02T22:14:08,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T22:14:08,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/8981900ebdc44daf8b98011da13fec3a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a 2024-12-02T22:14:08,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T22:14:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/d7c74966e29c435b805ada5381413dda as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda 2024-12-02T22:14:08,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T22:14:08,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8de9171fd3e9b964cfb173aae742ae78 in 540ms, sequenceid=374, compaction requested=true 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:08,458 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:08,458 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:08,460 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:08,460 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:08,460 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:08,460 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:08,460 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,460 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,460 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/1376b8d434e84898b68e6af778d6895c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.8 K 2024-12-02T22:14:08,460 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/583cff4e216843698b4f433afcd42ab9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.8 K 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1376b8d434e84898b68e6af778d6895c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646032 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583cff4e216843698b4f433afcd42ab9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646032 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1017c3bd644eba9bd2aeca4ee44e9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733177646711 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9406390b34f4350ad61309caca77820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733177646711 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8981900ebdc44daf8b98011da13fec3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:08,461 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02724b6b2964e61a53cee971a24d592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:08,485 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:08,485 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:08,486 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/71e8397b7c22431591a2ea35ac37aaef is 50, key is test_row_0/A:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:08,486 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/ca5a81c9e15f4074b8f08761f7c2d5b6 is 50, key is test_row_0/B:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:08,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741915_1091 (size=13187) 2024-12-02T22:14:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741916_1092 (size=13187) 2024-12-02T22:14:08,508 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/71e8397b7c22431591a2ea35ac37aaef as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/71e8397b7c22431591a2ea35ac37aaef 2024-12-02T22:14:08,516 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/ca5a81c9e15f4074b8f08761f7c2d5b6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ca5a81c9e15f4074b8f08761f7c2d5b6 2024-12-02T22:14:08,530 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into ca5a81c9e15f4074b8f08761f7c2d5b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:08,530 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into 71e8397b7c22431591a2ea35ac37aaef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:08,531 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:08,531 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:08,531 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177648458; duration=0sec 2024-12-02T22:14:08,531 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177648458; duration=0sec 2024-12-02T22:14:08,531 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:08,531 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:08,531 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:08,532 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:08,532 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:08,533 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:08,533 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:08,533 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,533 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9ec4d11cb3954d8e8a31575e5039c717, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.8 K 2024-12-02T22:14:08,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec4d11cb3954d8e8a31575e5039c717, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733177646032 2024-12-02T22:14:08,535 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c7d21c64744b75a295dd896b75b1ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733177646711 2024-12-02T22:14:08,535 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c74966e29c435b805ada5381413dda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:08,550 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:08,550 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e is 50, key is test_row_0/C:col10/1733177647867/Put/seqid=0 2024-12-02T22:14:08,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741917_1093 (size=13187) 2024-12-02T22:14:08,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T22:14:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:08,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f603fb410580445492116beb2308adf1 is 50, key is test_row_0/A:col10/1733177648055/Put/seqid=0 2024-12-02T22:14:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741918_1094 (size=12301) 2024-12-02T22:14:08,614 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f603fb410580445492116beb2308adf1 2024-12-02T22:14:08,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e6c65dc392604fb6bb73ce83134a97a3 is 50, key is test_row_0/B:col10/1733177648055/Put/seqid=0 2024-12-02T22:14:08,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741919_1095 (size=12301) 2024-12-02T22:14:08,675 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e6c65dc392604fb6bb73ce83134a97a3 2024-12-02T22:14:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/36ece6dccaa2444e8d98e7bc571e1aed is 50, key is test_row_0/C:col10/1733177648055/Put/seqid=0 2024-12-02T22:14:08,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:08,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741920_1096 (size=12301) 2024-12-02T22:14:08,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177708716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177708716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177708718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T22:14:08,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177708833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177708833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177708833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:08,968 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e 2024-12-02T22:14:08,978 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into 04b8a5d24b0b4d7385ec0e7a92aa3b3e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:08,978 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:08,978 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177648458; duration=0sec 2024-12-02T22:14:08,978 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:08,978 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:09,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177709043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177709054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177709054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38458 deadline: 1733177709083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,090 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38448 deadline: 1733177709103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,108 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 8de9171fd3e9b964cfb173aae742ae78 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/36ece6dccaa2444e8d98e7bc571e1aed 2024-12-02T22:14:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f603fb410580445492116beb2308adf1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1 2024-12-02T22:14:09,125 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1, entries=150, sequenceid=411, filesize=12.0 K 2024-12-02T22:14:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/e6c65dc392604fb6bb73ce83134a97a3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3 2024-12-02T22:14:09,131 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3, entries=150, sequenceid=411, filesize=12.0 K 2024-12-02T22:14:09,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/36ece6dccaa2444e8d98e7bc571e1aed as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed 2024-12-02T22:14:09,138 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed, entries=150, sequenceid=411, filesize=12.0 K 2024-12-02T22:14:09,140 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 8de9171fd3e9b964cfb173aae742ae78 in 559ms, sequenceid=411, compaction requested=false 2024-12-02T22:14:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T22:14:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T22:14:09,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T22:14:09,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0300 sec 2024-12-02T22:14:09,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0410 sec 2024-12-02T22:14:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T22:14:09,244 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:09,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-02T22:14:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:09,247 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:09,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:09,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:09,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T22:14:09,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:09,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:09,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:09,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:09,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:09,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:09,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e4f393a94cce44368a37c3161f2382a6 is 50, key is test_row_0/A:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741921_1097 (size=14741) 2024-12-02T22:14:09,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e4f393a94cce44368a37c3161f2382a6 2024-12-02T22:14:09,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/37eed18b6f244cf7a996cd41c85224eb is 50, key is test_row_0/B:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:09,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741922_1098 (size=12301) 2024-12-02T22:14:09,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:09,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:09,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177709399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177709399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177709401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177709513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177709513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177709514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:09,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:09,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:09,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177709724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177709724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177709724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:09,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/37eed18b6f244cf7a996cd41c85224eb 2024-12-02T22:14:09,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/92528cbe9c0e41629a87d0342f63d982 is 50, key is test_row_0/C:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:09,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741923_1099 (size=12301) 2024-12-02T22:14:09,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:09,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:09,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:09,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:10,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:10,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177710033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177710034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177710034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:10,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/92528cbe9c0e41629a87d0342f63d982 2024-12-02T22:14:10,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/e4f393a94cce44368a37c3161f2382a6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6 2024-12-02T22:14:10,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6, entries=200, sequenceid=424, filesize=14.4 K 2024-12-02T22:14:10,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/37eed18b6f244cf7a996cd41c85224eb as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb 2024-12-02T22:14:10,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb, entries=150, sequenceid=424, filesize=12.0 K 2024-12-02T22:14:10,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/92528cbe9c0e41629a87d0342f63d982 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982 2024-12-02T22:14:10,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982, entries=150, sequenceid=424, filesize=12.0 K 2024-12-02T22:14:10,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8de9171fd3e9b964cfb173aae742ae78 in 899ms, sequenceid=424, compaction requested=true 2024-12-02T22:14:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:10,255 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:10,256 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:10,257 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:10,257 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:10,258 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,258 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/71e8397b7c22431591a2ea35ac37aaef, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=39.3 K 2024-12-02T22:14:10,258 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:10,258 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:10,258 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,258 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ca5a81c9e15f4074b8f08761f7c2d5b6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.9 K 2024-12-02T22:14:10,259 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e8397b7c22431591a2ea35ac37aaef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:10,259 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5a81c9e15f4074b8f08761f7c2d5b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:10,259 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f603fb410580445492116beb2308adf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177648051 2024-12-02T22:14:10,260 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c65dc392604fb6bb73ce83134a97a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177648051 2024-12-02T22:14:10,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f393a94cce44368a37c3161f2382a6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:10,263 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 37eed18b6f244cf7a996cd41c85224eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:10,281 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:10,282 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:10,282 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/0c8cbdf4aa2e4003bd37863530252d32 is 50, key is test_row_0/A:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:10,282 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/32346723aab14d8eae952c445347ef92 is 50, key is test_row_0/B:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:10,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741924_1100 (size=13289) 2024-12-02T22:14:10,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741925_1101 (size=13289) 2024-12-02T22:14:10,309 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/32346723aab14d8eae952c445347ef92 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/32346723aab14d8eae952c445347ef92 2024-12-02T22:14:10,311 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/0c8cbdf4aa2e4003bd37863530252d32 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/0c8cbdf4aa2e4003bd37863530252d32 2024-12-02T22:14:10,317 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 32346723aab14d8eae952c445347ef92(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:10,317 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:10,317 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177650255; duration=0sec 2024-12-02T22:14:10,317 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:10,317 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:10,317 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:10,318 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:10,318 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:10,318 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,319 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=36.9 K 2024-12-02T22:14:10,319 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b8a5d24b0b4d7385ec0e7a92aa3b3e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177647867 2024-12-02T22:14:10,320 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ece6dccaa2444e8d98e7bc571e1aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177648051 2024-12-02T22:14:10,322 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 92528cbe9c0e41629a87d0342f63d982, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:10,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:10,323 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:10,327 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into 0c8cbdf4aa2e4003bd37863530252d32(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:10,327 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:10,327 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177650255; duration=0sec 2024-12-02T22:14:10,327 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:10,327 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:10,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1c40febec7ee41ad8eef501166fa38ad is 50, key is test_row_0/A:col10/1733177649398/Put/seqid=0 2024-12-02T22:14:10,338 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:10,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741926_1102 (size=12301) 2024-12-02T22:14:10,349 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/c631186174cd455d8cac25d5fa8295df is 50, key is test_row_0/C:col10/1733177648714/Put/seqid=0 2024-12-02T22:14:10,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741927_1103 (size=13289) 2024-12-02T22:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:10,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. as already flushing 2024-12-02T22:14:10,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177710569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177710574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177710575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177710683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177710693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177710693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1c40febec7ee41ad8eef501166fa38ad 2024-12-02T22:14:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/f7980410a14e4855b5c417d2cf4ac441 is 50, key is test_row_0/B:col10/1733177649398/Put/seqid=0 2024-12-02T22:14:10,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741928_1104 (size=12301) 2024-12-02T22:14:10,776 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/c631186174cd455d8cac25d5fa8295df as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c631186174cd455d8cac25d5fa8295df 2024-12-02T22:14:10,782 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into c631186174cd455d8cac25d5fa8295df(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:10,782 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:10,783 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177650256; duration=0sec 2024-12-02T22:14:10,783 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:10,783 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177710893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177710903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:10,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177710904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,161 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/f7980410a14e4855b5c417d2cf4ac441 2024-12-02T22:14:11,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ec31a01382964d6bbd9a0cc534344439 is 50, key is test_row_0/C:col10/1733177649398/Put/seqid=0 2024-12-02T22:14:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741929_1105 (size=12301) 2024-12-02T22:14:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177711203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177711213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177711213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:11,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ec31a01382964d6bbd9a0cc534344439 2024-12-02T22:14:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/1c40febec7ee41ad8eef501166fa38ad as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad 2024-12-02T22:14:11,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T22:14:11,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/f7980410a14e4855b5c417d2cf4ac441 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441 2024-12-02T22:14:11,606 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T22:14:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ec31a01382964d6bbd9a0cc534344439 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439 2024-12-02T22:14:11,616 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T22:14:11,617 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8de9171fd3e9b964cfb173aae742ae78 in 1294ms, sequenceid=446, compaction requested=false 2024-12-02T22:14:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-02T22:14:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-02T22:14:11,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T22:14:11,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-12-02T22:14:11,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3780 sec 2024-12-02T22:14:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:11,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:11,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/436d2189eff047e6bd6975b0125fd724 is 50, key is test_row_0/A:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:11,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741930_1106 (size=14741) 2024-12-02T22:14:11,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177711765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177711766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177711767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,855 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-02T22:14:11,855 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-02T22:14:11,855 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:11,855 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:11,857 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-02T22:14:11,857 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:11,859 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-02T22:14:11,859 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:11,860 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,860 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,860 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,860 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,861 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,861 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:11,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:11,861 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:11,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:11,862 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177711873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177711873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:11,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177711883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177712083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177712083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177712093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/436d2189eff047e6bd6975b0125fd724 2024-12-02T22:14:12,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/032b7aedde7d4c7bb6a27781d6603aa3 is 50, key is test_row_0/B:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:12,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741931_1107 (size=12301) 2024-12-02T22:14:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177712393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177712393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177712403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/032b7aedde7d4c7bb6a27781d6603aa3 2024-12-02T22:14:12,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/102d074eea6b4d7f81bbdcd821003f97 is 50, key is test_row_0/C:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:12,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741932_1108 (size=12301) 2024-12-02T22:14:12,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38494 deadline: 1733177712903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38436 deadline: 1733177712903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38472 deadline: 1733177712913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:12,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/102d074eea6b4d7f81bbdcd821003f97 2024-12-02T22:14:12,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/436d2189eff047e6bd6975b0125fd724 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724 2024-12-02T22:14:12,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724, entries=200, sequenceid=460, filesize=14.4 K 2024-12-02T22:14:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/032b7aedde7d4c7bb6a27781d6603aa3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3 2024-12-02T22:14:12,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3, entries=150, sequenceid=460, filesize=12.0 K 2024-12-02T22:14:12,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/102d074eea6b4d7f81bbdcd821003f97 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97 2024-12-02T22:14:12,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97, entries=150, sequenceid=460, filesize=12.0 K 2024-12-02T22:14:12,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8de9171fd3e9b964cfb173aae742ae78 in 1269ms, sequenceid=460, compaction requested=true 2024-12-02T22:14:12,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:12,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:12,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8de9171fd3e9b964cfb173aae742ae78:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:12,985 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:12,986 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:12,986 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/A is initiating minor compaction (all files) 2024-12-02T22:14:12,986 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/A in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:12,986 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/0c8cbdf4aa2e4003bd37863530252d32, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=39.4 K 2024-12-02T22:14:12,986 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:12,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8cbdf4aa2e4003bd37863530252d32, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:12,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c40febec7ee41ad8eef501166fa38ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733177649396 2024-12-02T22:14:12,988 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436d2189eff047e6bd6975b0125fd724, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733177650567 2024-12-02T22:14:12,988 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:12,988 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/B is initiating minor compaction (all files) 2024-12-02T22:14:12,988 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/B in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:12,988 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/32346723aab14d8eae952c445347ef92, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=37.0 K 2024-12-02T22:14:12,989 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 32346723aab14d8eae952c445347ef92, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:12,989 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f7980410a14e4855b5c417d2cf4ac441, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733177649396 2024-12-02T22:14:12,990 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 032b7aedde7d4c7bb6a27781d6603aa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733177650567 2024-12-02T22:14:12,997 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:12,998 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f9fa7f36fa374414b5bcca112f1c071f is 50, key is test_row_0/A:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:13,003 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:13,003 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/9fc94cafabc8445f91b7652c3a094128 is 50, key is test_row_0/B:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:13,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741933_1109 (size=13391) 2024-12-02T22:14:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741934_1110 (size=13391) 2024-12-02T22:14:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:13,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:13,116 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:13,116 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:13,116 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:13,116 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a98ce99ce98c4df79223c9e018e9d20f is 50, key is test_row_0/A:col10/1733177651765/Put/seqid=0 2024-12-02T22:14:13,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741935_1111 (size=12301) 2024-12-02T22:14:13,145 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-02T22:14:13,145 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:13,145 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,145 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,411 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/f9fa7f36fa374414b5bcca112f1c071f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f9fa7f36fa374414b5bcca112f1c071f 2024-12-02T22:14:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T22:14:13,413 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:13,415 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/9fc94cafabc8445f91b7652c3a094128 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9fc94cafabc8445f91b7652c3a094128 2024-12-02T22:14:13,416 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/A of 8de9171fd3e9b964cfb173aae742ae78 into f9fa7f36fa374414b5bcca112f1c071f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:13,416 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:13,416 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/A, priority=13, startTime=1733177652984; duration=0sec 2024-12-02T22:14:13,417 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:13,417 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:A 2024-12-02T22:14:13,417 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:13,418 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:13,418 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 8de9171fd3e9b964cfb173aae742ae78/C is initiating minor compaction (all files) 2024-12-02T22:14:13,418 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8de9171fd3e9b964cfb173aae742ae78/C in TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:13,418 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c631186174cd455d8cac25d5fa8295df, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp, totalSize=37.0 K 2024-12-02T22:14:13,419 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c631186174cd455d8cac25d5fa8295df, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733177648714 2024-12-02T22:14:13,419 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec31a01382964d6bbd9a0cc534344439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733177649396 2024-12-02T22:14:13,420 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102d074eea6b4d7f81bbdcd821003f97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733177650567 2024-12-02T22:14:13,420 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/B of 8de9171fd3e9b964cfb173aae742ae78 into 9fc94cafabc8445f91b7652c3a094128(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:13,420 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:13,420 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/B, priority=13, startTime=1733177652984; duration=0sec 2024-12-02T22:14:13,421 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:13,421 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:B 2024-12-02T22:14:13,428 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8de9171fd3e9b964cfb173aae742ae78#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:13,429 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/babed32459294113aeaa77cccae84602 is 50, key is test_row_0/C:col10/1733177650573/Put/seqid=0 2024-12-02T22:14:13,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741936_1112 (size=13391) 2024-12-02T22:14:13,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a98ce99ce98c4df79223c9e018e9d20f 2024-12-02T22:14:13,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/017419565f0f4a1fbe85c8f57e3d082a is 50, key is test_row_0/B:col10/1733177651765/Put/seqid=0 2024-12-02T22:14:13,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741937_1113 (size=12301) 2024-12-02T22:14:13,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/017419565f0f4a1fbe85c8f57e3d082a 2024-12-02T22:14:13,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ea1ad43956174399a90cce9d1e94160e is 50, key is test_row_0/C:col10/1733177651765/Put/seqid=0 2024-12-02T22:14:13,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741938_1114 (size=12301) 2024-12-02T22:14:13,840 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/babed32459294113aeaa77cccae84602 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/babed32459294113aeaa77cccae84602 2024-12-02T22:14:13,846 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8de9171fd3e9b964cfb173aae742ae78/C of 8de9171fd3e9b964cfb173aae742ae78 into babed32459294113aeaa77cccae84602(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:13,846 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:13,846 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., storeName=8de9171fd3e9b964cfb173aae742ae78/C, priority=13, startTime=1733177652985; duration=0sec 2024-12-02T22:14:13,846 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:13,846 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8de9171fd3e9b964cfb173aae742ae78:C 2024-12-02T22:14:13,915 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-02T22:14:13,915 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:13,915 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,915 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,925 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-02T22:14:13,925 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:13,925 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,925 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,926 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-02T22:14:13,926 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:13,926 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,926 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-02T22:14:13,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2379 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2390 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1091 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3272 rows 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1093 2024-12-02T22:14:13,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3278 rows 2024-12-02T22:14:13,927 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:14:13,927 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:14:13,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:13,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:13,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:14:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:13,941 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177653940"}]},"ts":"1733177653940"} 2024-12-02T22:14:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:13,943 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:14:13,943 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:14:13,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:14:13,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, UNASSIGN}] 2024-12-02T22:14:13,950 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, UNASSIGN 2024-12-02T22:14:13,951 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8de9171fd3e9b964cfb173aae742ae78, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:13,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, UNASSIGN because future has completed 2024-12-02T22:14:13,953 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:14:13,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:13,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ea1ad43956174399a90cce9d1e94160e 2024-12-02T22:14:13,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/a98ce99ce98c4df79223c9e018e9d20f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a98ce99ce98c4df79223c9e018e9d20f 2024-12-02T22:14:13,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a98ce99ce98c4df79223c9e018e9d20f, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T22:14:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/017419565f0f4a1fbe85c8f57e3d082a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/017419565f0f4a1fbe85c8f57e3d082a 2024-12-02T22:14:13,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/017419565f0f4a1fbe85c8f57e3d082a, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T22:14:13,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/ea1ad43956174399a90cce9d1e94160e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ea1ad43956174399a90cce9d1e94160e 2024-12-02T22:14:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ea1ad43956174399a90cce9d1e94160e, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T22:14:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for 8de9171fd3e9b964cfb173aae742ae78 in 868ms, sequenceid=480, compaction requested=false 2024-12-02T22:14:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8de9171fd3e9b964cfb173aae742ae78: 2024-12-02T22:14:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:14,109 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:14,110 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:14:14,110 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 8de9171fd3e9b964cfb173aae742ae78, disabling compactions & flushes 2024-12-02T22:14:14,110 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:14,110 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:14,110 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. after waiting 0 ms 2024-12-02T22:14:14,110 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:14,110 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 8de9171fd3e9b964cfb173aae742ae78 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=A 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=B 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8de9171fd3e9b964cfb173aae742ae78, store=C 2024-12-02T22:14:14,111 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:14,114 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/9a714ef1f34d4f578688a8f55e7d97e4 is 50, key is test_row_0/A:col10/1733177653924/Put/seqid=0 2024-12-02T22:14:14,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741939_1115 (size=12301) 2024-12-02T22:14:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:14,519 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/9a714ef1f34d4f578688a8f55e7d97e4 2024-12-02T22:14:14,529 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/40f1545696314d3dabb8ae848d9587be is 50, key is test_row_0/B:col10/1733177653924/Put/seqid=0 2024-12-02T22:14:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741940_1116 (size=12301) 2024-12-02T22:14:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:14,934 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/40f1545696314d3dabb8ae848d9587be 2024-12-02T22:14:14,942 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/5d6c9cc35d794830a6d1e5f006652127 is 50, key is test_row_0/C:col10/1733177653924/Put/seqid=0 2024-12-02T22:14:14,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741941_1117 (size=12301) 2024-12-02T22:14:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:15,347 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/5d6c9cc35d794830a6d1e5f006652127 2024-12-02T22:14:15,353 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/A/9a714ef1f34d4f578688a8f55e7d97e4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/9a714ef1f34d4f578688a8f55e7d97e4 2024-12-02T22:14:15,357 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/9a714ef1f34d4f578688a8f55e7d97e4, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T22:14:15,358 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/B/40f1545696314d3dabb8ae848d9587be as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/40f1545696314d3dabb8ae848d9587be 2024-12-02T22:14:15,363 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/40f1545696314d3dabb8ae848d9587be, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T22:14:15,364 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/.tmp/C/5d6c9cc35d794830a6d1e5f006652127 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/5d6c9cc35d794830a6d1e5f006652127 2024-12-02T22:14:15,370 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/5d6c9cc35d794830a6d1e5f006652127, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T22:14:15,370 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8de9171fd3e9b964cfb173aae742ae78 in 1260ms, sequenceid=490, compaction requested=true 2024-12-02T22:14:15,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a4049a7ea3594ba4ac96a094ad52de67, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f6fe5b934a6d4dea81b053acd884e923, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/4a360754d110494faf9402dd222c0476, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b2e6acab364044e2b41193f2d84a914c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/3828f294dbf54e6cb148e08fb7ffcfc7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/feb0d0f68e1c4b1daa611b0c48b34df6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/583cff4e216843698b4f433afcd42ab9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/71e8397b7c22431591a2ea35ac37aaef, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/0c8cbdf4aa2e4003bd37863530252d32, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724] to archive 2024-12-02T22:14:15,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:15,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/d610823a67d84135b15adc57840aa4fc 2024-12-02T22:14:15,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/ad9998f1909e465faf555ae59b94cd06 2024-12-02T22:14:15,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a4049a7ea3594ba4ac96a094ad52de67 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a4049a7ea3594ba4ac96a094ad52de67 2024-12-02T22:14:15,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/c5c0ceb5f4f24e36bed59546f873402a 2024-12-02T22:14:15,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1da87f06eefa4da98c37c12a7ca282cf 2024-12-02T22:14:15,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f6fe5b934a6d4dea81b053acd884e923 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f6fe5b934a6d4dea81b053acd884e923 2024-12-02T22:14:15,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/62237694ca5c481d8987f74db3447d69 2024-12-02T22:14:15,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:14:15,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/fa969eefb40e437bb413bacbfa3067e1 2024-12-02T22:14:15,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/4a360754d110494faf9402dd222c0476 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/4a360754d110494faf9402dd222c0476 2024-12-02T22:14:15,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e3bfc8f847c44fceadccea8f176a461b 2024-12-02T22:14:15,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/2a30dcf5663d407c8a17cccf755dea73 2024-12-02T22:14:15,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b114bd26a51c4d68a87ef8b66212f5df 2024-12-02T22:14:15,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b2e6acab364044e2b41193f2d84a914c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b2e6acab364044e2b41193f2d84a914c 2024-12-02T22:14:15,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/6c37254e6a9b4735869d7881890ad4f2 2024-12-02T22:14:15,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/3828f294dbf54e6cb148e08fb7ffcfc7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/3828f294dbf54e6cb148e08fb7ffcfc7 2024-12-02T22:14:15,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/579f1c1f38d940cba7ddc1f45d779a49 2024-12-02T22:14:15,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/982da7575b2740d393e6a82e3c0f5bd7 2024-12-02T22:14:15,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/08fbf3e0ffc74f5cbd578302aa875a33 2024-12-02T22:14:15,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/feb0d0f68e1c4b1daa611b0c48b34df6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/feb0d0f68e1c4b1daa611b0c48b34df6 2024-12-02T22:14:15,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/97e736e383d94ff398c52883898b421e 2024-12-02T22:14:15,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/5b4e9d94438844dab565b7b5a79b82a1 2024-12-02T22:14:15,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/715709401bff46a1ad99507c431b640d 2024-12-02T22:14:15,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a6da67afb2cb460bb9db74ac651be847 2024-12-02T22:14:15,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/583cff4e216843698b4f433afcd42ab9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/583cff4e216843698b4f433afcd42ab9 2024-12-02T22:14:15,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/b9406390b34f4350ad61309caca77820 2024-12-02T22:14:15,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/71e8397b7c22431591a2ea35ac37aaef to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/71e8397b7c22431591a2ea35ac37aaef 2024-12-02T22:14:15,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f02724b6b2964e61a53cee971a24d592 2024-12-02T22:14:15,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f603fb410580445492116beb2308adf1 2024-12-02T22:14:15,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/e4f393a94cce44368a37c3161f2382a6 2024-12-02T22:14:15,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/0c8cbdf4aa2e4003bd37863530252d32 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/0c8cbdf4aa2e4003bd37863530252d32 2024-12-02T22:14:15,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/1c40febec7ee41ad8eef501166fa38ad 2024-12-02T22:14:15,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/436d2189eff047e6bd6975b0125fd724 2024-12-02T22:14:15,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d791b9ba5819476dbeed3cb0eded0e32, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d303102ca3a548fbb591dbbe0edea178, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/058592f596874187be5ee5d624fc7150, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/176a2eafd0f6456680c4ae183256c6c4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/22a1bfd218f84f50b0cbfed12b70ea3d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/831d9735c1904214bb183c96389c0d1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/1376b8d434e84898b68e6af778d6895c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ca5a81c9e15f4074b8f08761f7c2d5b6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/32346723aab14d8eae952c445347ef92, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3] to archive 2024-12-02T22:14:15,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:15,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/eec0a457ea454be586e946184ad40eca 2024-12-02T22:14:15,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/df834ccc12544d8c9930c5f0a6d39950 2024-12-02T22:14:15,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d791b9ba5819476dbeed3cb0eded0e32 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d791b9ba5819476dbeed3cb0eded0e32 2024-12-02T22:14:15,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/4c765d694d7c40608057c8589605d175 2024-12-02T22:14:15,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6065d71d30c34f8baab2077d3f75c4bd 2024-12-02T22:14:15,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d303102ca3a548fbb591dbbe0edea178 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/d303102ca3a548fbb591dbbe0edea178 2024-12-02T22:14:15,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3befd9e48f0f40b88ad3e8660776eb27 2024-12-02T22:14:15,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/76fc0e5972ed4b578373a499ee3af6c1 2024-12-02T22:14:15,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/058592f596874187be5ee5d624fc7150 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/058592f596874187be5ee5d624fc7150 2024-12-02T22:14:15,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3dd032787ad64177858ad954f6c77f29 2024-12-02T22:14:15,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/854fe06294474bbda312533d5fc9c330 2024-12-02T22:14:15,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/176a2eafd0f6456680c4ae183256c6c4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/176a2eafd0f6456680c4ae183256c6c4 2024-12-02T22:14:15,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/6fad2752cdb64947ac073733ab2acfa0 2024-12-02T22:14:15,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/54fb87088ec2495580d670ebb3196cf6 2024-12-02T22:14:15,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/22a1bfd218f84f50b0cbfed12b70ea3d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/22a1bfd218f84f50b0cbfed12b70ea3d 2024-12-02T22:14:15,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/7354a9d74b764a0fbe1aaca071b60b82 2024-12-02T22:14:15,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9e7dbee483dc4f16bc6a4367346aa350 2024-12-02T22:14:15,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/831d9735c1904214bb183c96389c0d1f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/831d9735c1904214bb183c96389c0d1f 2024-12-02T22:14:15,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/256463f823fc4ddb84f3687db027af73 2024-12-02T22:14:15,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8628ca5a2f35448191783692e8223d88 2024-12-02T22:14:15,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/3f99da527edf451583a0e9ee43bb9c92 2024-12-02T22:14:15,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ad38ba254cff4afba4b484a087dde5b9 2024-12-02T22:14:15,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/1376b8d434e84898b68e6af778d6895c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/1376b8d434e84898b68e6af778d6895c 2024-12-02T22:14:15,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e495e3a7fe7741ceac0433ce62c3c4e2 2024-12-02T22:14:15,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/0c1017c3bd644eba9bd2aeca4ee44e9e 2024-12-02T22:14:15,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ca5a81c9e15f4074b8f08761f7c2d5b6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/ca5a81c9e15f4074b8f08761f7c2d5b6 2024-12-02T22:14:15,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/8981900ebdc44daf8b98011da13fec3a 2024-12-02T22:14:15,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/e6c65dc392604fb6bb73ce83134a97a3 2024-12-02T22:14:15,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/32346723aab14d8eae952c445347ef92 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/32346723aab14d8eae952c445347ef92 2024-12-02T22:14:15,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/37eed18b6f244cf7a996cd41c85224eb 2024-12-02T22:14:15,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/f7980410a14e4855b5c417d2cf4ac441 2024-12-02T22:14:15,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/032b7aedde7d4c7bb6a27781d6603aa3 2024-12-02T22:14:15,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/672a9a44898b4c5d874fc73959ab8ebc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d62f069c4c0e4a4796d93dfab2d9a605, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/aa9c1b746b7346feab29c8baeec33ab1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e3ad28f1c9b04946ab129cd54270770e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/909e8185adc546b38eee0e1e141cdc99, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/906740eb10ad48dcb216f32425ac9c1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9ec4d11cb3954d8e8a31575e5039c717, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c631186174cd455d8cac25d5fa8295df, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97] to archive 2024-12-02T22:14:15,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:15,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/99bc828807764794a7aba8cac425322a 2024-12-02T22:14:15,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e0695668db9a4d728eb81ea61bb22730 2024-12-02T22:14:15,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/672a9a44898b4c5d874fc73959ab8ebc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/672a9a44898b4c5d874fc73959ab8ebc 2024-12-02T22:14:15,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/2d49a101e111450d84ed0a90e6a6a2dd 2024-12-02T22:14:15,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/67362084d7074bdbb25d26ec28bab517 2024-12-02T22:14:15,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d62f069c4c0e4a4796d93dfab2d9a605 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d62f069c4c0e4a4796d93dfab2d9a605 2024-12-02T22:14:15,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/06128674cb564c238c893db21d4ff556 2024-12-02T22:14:15,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/0eb2706fe47b43a698413b8f9608ec4e 2024-12-02T22:14:15,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/aa9c1b746b7346feab29c8baeec33ab1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/aa9c1b746b7346feab29c8baeec33ab1 2024-12-02T22:14:15,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/db4086d7fbb44e0783d1c5bbfc2728e7 2024-12-02T22:14:15,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9aacdc50afa0447c86635e305d6a1ec0 2024-12-02T22:14:15,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e3ad28f1c9b04946ab129cd54270770e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/e3ad28f1c9b04946ab129cd54270770e 2024-12-02T22:14:15,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/debded6d84b547019ed3e3c6ef630f86 2024-12-02T22:14:15,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ae256155f1d045c092b6c321e51565e4 2024-12-02T22:14:15,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/909e8185adc546b38eee0e1e141cdc99 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/909e8185adc546b38eee0e1e141cdc99 2024-12-02T22:14:15,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/3e187f7e98b44030b3f705e135fdb28b 2024-12-02T22:14:15,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/6ebe5ebb289d48c8a3fae27ab347eb67 2024-12-02T22:14:15,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/906740eb10ad48dcb216f32425ac9c1f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/906740eb10ad48dcb216f32425ac9c1f 2024-12-02T22:14:15,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/a9b5fdaff91841d59994072926e69a50 2024-12-02T22:14:15,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/4f492cb7feb049388dc3a0f29775dec5 2024-12-02T22:14:15,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ef72f422e97945539a5f568e3d45bc06 2024-12-02T22:14:15,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c3e583af21744a5c9779b0ef8f538aa2 2024-12-02T22:14:15,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9ec4d11cb3954d8e8a31575e5039c717 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/9ec4d11cb3954d8e8a31575e5039c717 2024-12-02T22:14:15,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/60da36b2551a478b8a834fd25e77510a 2024-12-02T22:14:15,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/f2c7d21c64744b75a295dd896b75b1ff 2024-12-02T22:14:15,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/04b8a5d24b0b4d7385ec0e7a92aa3b3e 2024-12-02T22:14:15,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/d7c74966e29c435b805ada5381413dda 2024-12-02T22:14:15,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/36ece6dccaa2444e8d98e7bc571e1aed 2024-12-02T22:14:15,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c631186174cd455d8cac25d5fa8295df to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/c631186174cd455d8cac25d5fa8295df 2024-12-02T22:14:15,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/92528cbe9c0e41629a87d0342f63d982 2024-12-02T22:14:15,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ec31a01382964d6bbd9a0cc534344439 2024-12-02T22:14:15,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/102d074eea6b4d7f81bbdcd821003f97 2024-12-02T22:14:15,523 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/recovered.edits/493.seqid, newMaxSeqId=493, maxSeqId=1 2024-12-02T22:14:15,523 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. 2024-12-02T22:14:15,524 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 8de9171fd3e9b964cfb173aae742ae78: Waiting for close lock at 1733177654110Running coprocessor pre-close hooks at 1733177654110Disabling compacts and flushes for region at 1733177654110Disabling writes for close at 1733177654110Obtaining lock to block concurrent updates at 1733177654110Preparing flush snapshotting stores in 8de9171fd3e9b964cfb173aae742ae78 at 1733177654110Finished memstore snapshotting TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733177654111 (+1 ms)Flushing stores of TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78. at 1733177654111Flushing 8de9171fd3e9b964cfb173aae742ae78/A: creating writer at 1733177654111Flushing 8de9171fd3e9b964cfb173aae742ae78/A: appending metadata at 1733177654114 (+3 ms)Flushing 8de9171fd3e9b964cfb173aae742ae78/A: closing flushed file at 1733177654114Flushing 8de9171fd3e9b964cfb173aae742ae78/B: creating writer at 1733177654525 (+411 ms)Flushing 8de9171fd3e9b964cfb173aae742ae78/B: appending metadata at 1733177654529 (+4 ms)Flushing 8de9171fd3e9b964cfb173aae742ae78/B: closing flushed file at 1733177654529Flushing 8de9171fd3e9b964cfb173aae742ae78/C: creating writer at 1733177654939 (+410 ms)Flushing 8de9171fd3e9b964cfb173aae742ae78/C: appending metadata at 1733177654942 (+3 ms)Flushing 8de9171fd3e9b964cfb173aae742ae78/C: closing flushed file at 1733177654942Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18eaa4b5: reopening flushed file at 1733177655352 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e294f14: reopening flushed file at 1733177655357 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2463289c: reopening flushed file at 1733177655363 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8de9171fd3e9b964cfb173aae742ae78 in 1260ms, sequenceid=490, compaction requested=true at 1733177655371 (+8 ms)Writing region close event to WAL at 1733177655519 (+148 ms)Running coprocessor post-close hooks at 1733177655523 (+4 ms)Closed at 1733177655523 2024-12-02T22:14:15,526 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:15,527 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8de9171fd3e9b964cfb173aae742ae78, regionState=CLOSED 2024-12-02T22:14:15,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:15,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-02T22:14:15,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 8de9171fd3e9b964cfb173aae742ae78, server=7ae73638d47e,41753,1733177627056 in 1.5760 sec 2024-12-02T22:14:15,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-02T22:14:15,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8de9171fd3e9b964cfb173aae742ae78, UNASSIGN in 1.5820 sec 2024-12-02T22:14:15,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T22:14:15,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-12-02T22:14:15,536 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177655536"}]},"ts":"1733177655536"} 2024-12-02T22:14:15,537 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:14:15,537 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:14:15,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6040 sec 2024-12-02T22:14:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T22:14:16,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:16,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:14:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,092 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T22:14:16,093 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,098 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:16,103 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/recovered.edits] 2024-12-02T22:14:16,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/9a714ef1f34d4f578688a8f55e7d97e4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/9a714ef1f34d4f578688a8f55e7d97e4 2024-12-02T22:14:16,108 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a98ce99ce98c4df79223c9e018e9d20f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/a98ce99ce98c4df79223c9e018e9d20f 2024-12-02T22:14:16,109 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f9fa7f36fa374414b5bcca112f1c071f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/A/f9fa7f36fa374414b5bcca112f1c071f 2024-12-02T22:14:16,112 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/017419565f0f4a1fbe85c8f57e3d082a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/017419565f0f4a1fbe85c8f57e3d082a 2024-12-02T22:14:16,113 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/40f1545696314d3dabb8ae848d9587be to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/40f1545696314d3dabb8ae848d9587be 2024-12-02T22:14:16,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9fc94cafabc8445f91b7652c3a094128 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/B/9fc94cafabc8445f91b7652c3a094128 2024-12-02T22:14:16,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/5d6c9cc35d794830a6d1e5f006652127 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/5d6c9cc35d794830a6d1e5f006652127 2024-12-02T22:14:16,119 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/babed32459294113aeaa77cccae84602 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/babed32459294113aeaa77cccae84602 2024-12-02T22:14:16,120 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ea1ad43956174399a90cce9d1e94160e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/C/ea1ad43956174399a90cce9d1e94160e 2024-12-02T22:14:16,123 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/recovered.edits/493.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78/recovered.edits/493.seqid 2024-12-02T22:14:16,124 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/8de9171fd3e9b964cfb173aae742ae78 2024-12-02T22:14:16,124 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:14:16,129 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T22:14:16,137 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:14:16,172 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:14:16,174 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,174 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:14:16,174 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177656174"}]},"ts":"9223372036854775807"} 2024-12-02T22:14:16,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:14:16,179 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8de9171fd3e9b964cfb173aae742ae78, NAME => 'TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:14:16,179 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:14:16,179 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177656179"}]},"ts":"9223372036854775807"} 2024-12-02T22:14:16,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:14:16,183 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-12-02T22:14:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T22:14:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:14:16,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:16,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7ae73638d47e:41753-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1839004683_22 at /127.0.0.1:55704 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2041620475_22 at /127.0.0.1:55830 [Waiting for operation #316] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=523 (was 398) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7708 (was 8416) 2024-12-02T22:14:16,239 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=523, ProcessCount=11, AvailableMemoryMB=7704 2024-12-02T22:14:16,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:14:16,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:14:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,244 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:14:16,244 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:16,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-02T22:14:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T22:14:16,246 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:14:16,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741942_1118 (size=1061) 2024-12-02T22:14:16,277 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:14:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741943_1119 (size=53) 2024-12-02T22:14:16,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:14:16,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e76fa5b70a6b892d2b9930c0da4f98cd, disabling compactions & flushes 2024-12-02T22:14:16,292 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. after waiting 0 ms 2024-12-02T22:14:16,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,293 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,293 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e76fa5b70a6b892d2b9930c0da4f98cd: Waiting for close lock at 1733177656292Disabling compacts and flushes for region at 1733177656292Disabling writes for close at 1733177656292Writing region close event to WAL at 1733177656293 (+1 ms)Closed at 1733177656293 2024-12-02T22:14:16,294 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:14:16,295 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177656294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177656294"}]},"ts":"1733177656294"} 2024-12-02T22:14:16,297 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:14:16,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:14:16,299 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177656298"}]},"ts":"1733177656298"} 2024-12-02T22:14:16,300 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:14:16,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, ASSIGN}] 2024-12-02T22:14:16,302 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, ASSIGN 2024-12-02T22:14:16,303 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:14:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T22:14:16,453 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:16,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, ASSIGN because future has completed 2024-12-02T22:14:16,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T22:14:16,612 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,612 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:14:16,612 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,612 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:14:16,612 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,612 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,614 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,615 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:16,616 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName A 2024-12-02T22:14:16,616 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:16,616 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:16,617 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,618 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:16,618 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName B 2024-12-02T22:14:16,618 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:16,618 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:16,618 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,619 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:16,619 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName C 2024-12-02T22:14:16,619 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:16,620 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:16,620 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,620 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,620 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,621 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,622 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,622 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,623 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:14:16,624 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,626 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:14:16,626 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened e76fa5b70a6b892d2b9930c0da4f98cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74286776, jitterRate=0.10695922374725342}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:14:16,626 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:16,628 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for e76fa5b70a6b892d2b9930c0da4f98cd: Running coprocessor pre-open hook at 1733177656612Writing region info on filesystem at 1733177656612Initializing all the Stores at 1733177656613 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177656613Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177656614 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177656614Cleaning up temporary data from old regions at 1733177656622 (+8 ms)Running coprocessor post-open hooks at 1733177656627 (+5 ms)Region opened successfully at 1733177656628 (+1 ms) 2024-12-02T22:14:16,629 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., pid=34, masterSystemTime=1733177656608 2024-12-02T22:14:16,631 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,631 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:16,631 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:16,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:16,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-02T22:14:16,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 in 178 msec 2024-12-02T22:14:16,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-02T22:14:16,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, ASSIGN in 336 msec 2024-12-02T22:14:16,640 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:14:16,640 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177656640"}]},"ts":"1733177656640"} 2024-12-02T22:14:16,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:14:16,643 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:14:16,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 402 msec 2024-12-02T22:14:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T22:14:16,874 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:16,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:16,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:16,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:16,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9eb415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:16,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655d0d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:16,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:16,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:16,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:14:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:14:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ed75f2b 2024-12-02T22:14:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:14:16,888 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:14:16,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:14:16,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:14:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741944_1120 (size=1097) 2024-12-02T22:14:17,330 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T22:14:17,330 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T22:14:17,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:14:17,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE}] 2024-12-02T22:14:17,341 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE 2024-12-02T22:14:17,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:17,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE because future has completed 2024-12-02T22:14:17,344 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:14:17,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:17,496 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,496 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:14:17,496 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing e76fa5b70a6b892d2b9930c0da4f98cd, disabling compactions & flushes 2024-12-02T22:14:17,497 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,497 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,497 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. after waiting 0 ms 2024-12-02T22:14:17,497 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,501 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T22:14:17,501 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,501 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for e76fa5b70a6b892d2b9930c0da4f98cd: Waiting for close lock at 1733177657496Running coprocessor pre-close hooks at 1733177657496Disabling compacts and flushes for region at 1733177657496Disabling writes for close at 1733177657497 (+1 ms)Writing region close event to WAL at 1733177657497Running coprocessor post-close hooks at 1733177657501 (+4 ms)Closed at 1733177657501 2024-12-02T22:14:17,502 WARN [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: e76fa5b70a6b892d2b9930c0da4f98cd to self. 2024-12-02T22:14:17,503 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=CLOSED 2024-12-02T22:14:17,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:17,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-02T22:14:17,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 in 162 msec 2024-12-02T22:14:17,508 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE; state=CLOSED, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=true 2024-12-02T22:14:17,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:17,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE because future has completed 2024-12-02T22:14:17,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:17,818 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,818 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:14:17,818 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,818 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:14:17,818 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,818 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,823 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,824 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:17,828 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName A 2024-12-02T22:14:17,830 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:17,831 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:17,831 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,832 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:17,832 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName B 2024-12-02T22:14:17,832 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:17,832 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:17,832 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,833 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:17,833 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e76fa5b70a6b892d2b9930c0da4f98cd columnFamilyName C 2024-12-02T22:14:17,833 DEBUG [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:17,833 INFO [StoreOpener-e76fa5b70a6b892d2b9930c0da4f98cd-1 {}] regionserver.HStore(327): Store=e76fa5b70a6b892d2b9930c0da4f98cd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:17,834 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,834 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,834 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,836 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,837 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,837 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,837 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:14:17,838 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,839 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened e76fa5b70a6b892d2b9930c0da4f98cd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67512478, jitterRate=0.006014317274093628}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:14:17,839 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,840 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for e76fa5b70a6b892d2b9930c0da4f98cd: Running coprocessor pre-open hook at 1733177657818Writing region info on filesystem at 1733177657818Initializing all the Stores at 1733177657819 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177657820 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177657820Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177657820Cleaning up temporary data from old regions at 1733177657837 (+17 ms)Running coprocessor post-open hooks at 1733177657839 (+2 ms)Region opened successfully at 1733177657840 (+1 ms) 2024-12-02T22:14:17,841 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., pid=39, masterSystemTime=1733177657814 2024-12-02T22:14:17,843 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,843 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:17,844 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=OPEN, openSeqNum=5, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:17,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:17,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-02T22:14:17,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 in 186 msec 2024-12-02T22:14:17,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-02T22:14:17,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, REOPEN/MOVE in 509 msec 2024-12-02T22:14:17,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T22:14:17,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-12-02T22:14:17,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-12-02T22:14:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-02T22:14:17,859 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:17,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,862 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e88072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,864 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd7ed2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfd6758, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,871 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd6ad69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9c45d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e348d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae3a511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc7f85a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f369bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64445b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cacdda9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1166f8c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac8dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f899f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5fc009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:17,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:17,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5b0ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:17,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:17,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:17,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:17,914 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,914 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,914 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,914 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:17,915 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-02T22:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,917 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:17,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,918 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:17,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,930 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,930 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,931 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,932 DEBUG [Thread-608 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,933 DEBUG [Thread-608 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:17,972 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:17,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:14:17,975 DEBUG [Thread-614 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:17,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:17,977 DEBUG [Thread-614 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:17,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:14:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:18,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202419abc03377943c1af8726472505f37d_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177657973/Put/seqid=0 2024-12-02T22:14:18,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177658023, max timestamp=1733177658096Num uniques 859; 2024-12-02T22:14:18,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177658023, max timestamp=1733177658096Num uniques 859; 2024-12-02T22:14:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:18,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177658023, max timestamp=1733177658096Num uniques 859; 2024-12-02T22:14:18,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:18,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741945_1121 (size=14594) 2024-12-02T22:14:18,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177718165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177718181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177718181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177718183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177718184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177718295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177718303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177718303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177718304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177718304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,509 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:18,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202419abc03377943c1af8726472505f37d_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202419abc03377943c1af8726472505f37d_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:18,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/cdf357e0f0474b30beeeb5f6a6db4cdf, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:18,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/cdf357e0f0474b30beeeb5f6a6db4cdf is 175, key is test_row_0/A:col10/1733177657973/Put/seqid=0 2024-12-02T22:14:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177718524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177718525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177718525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177718525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177718525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:18,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741946_1122 (size=39549) 2024-12-02T22:14:18,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177718844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177718844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177718845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177718845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177718844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:18,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:18,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:18,865 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36511 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7ae73638d47e,41753,1733177627056, table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T22:14:18,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/cdf357e0f0474b30beeeb5f6a6db4cdf 2024-12-02T22:14:19,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:19,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:19,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b90bceba192941bc94817d00d39ff0da is 50, key is test_row_0/B:col10/1733177657973/Put/seqid=0 2024-12-02T22:14:19,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741947_1123 (size=12001) 2024-12-02T22:14:19,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b90bceba192941bc94817d00d39ff0da 2024-12-02T22:14:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:19,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/693afdf779cb4f6480a1f11f8708cc1b is 50, key is test_row_0/C:col10/1733177657973/Put/seqid=0 2024-12-02T22:14:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741948_1124 (size=12001) 2024-12-02T22:14:19,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/693afdf779cb4f6480a1f11f8708cc1b 2024-12-02T22:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/cdf357e0f0474b30beeeb5f6a6db4cdf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf 2024-12-02T22:14:19,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf, entries=200, sequenceid=16, filesize=38.6 K 2024-12-02T22:14:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b90bceba192941bc94817d00d39ff0da as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da 2024-12-02T22:14:19,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T22:14:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/693afdf779cb4f6480a1f11f8708cc1b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b 2024-12-02T22:14:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:19,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:19,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T22:14:19,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e76fa5b70a6b892d2b9930c0da4f98cd in 1206ms, sequenceid=16, compaction requested=false 2024-12-02T22:14:19,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:19,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T22:14:19,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:19,327 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120201e1085820474d3d90888b16ccf1a468_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177658180/Put/seqid=0 2024-12-02T22:14:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:19,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:19,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741949_1125 (size=12154) 2024-12-02T22:14:19,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177719382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177719385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177719400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,411 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120201e1085820474d3d90888b16ccf1a468_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120201e1085820474d3d90888b16ccf1a468_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7fde6e1692be4aa28c6ccf08934ed5db, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:19,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7fde6e1692be4aa28c6ccf08934ed5db is 175, key is test_row_0/A:col10/1733177658180/Put/seqid=0 2024-12-02T22:14:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177719406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177719407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741950_1126 (size=30955) 2024-12-02T22:14:19,433 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7fde6e1692be4aa28c6ccf08934ed5db 2024-12-02T22:14:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/86eb614d17a84ed6ac2dc00f20c2357a is 50, key is test_row_0/B:col10/1733177658180/Put/seqid=0 2024-12-02T22:14:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741951_1127 (size=12001) 2024-12-02T22:14:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177719513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177719514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177719514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177719523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177719533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177719734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177719734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177719735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177719735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177719754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:19,902 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/86eb614d17a84ed6ac2dc00f20c2357a 2024-12-02T22:14:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/29169608137e489eb32ff2c6efb9d5b1 is 50, key is test_row_0/C:col10/1733177658180/Put/seqid=0 2024-12-02T22:14:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741952_1128 (size=12001) 2024-12-02T22:14:20,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177720054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177720054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177720054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177720054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:20,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177720072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,385 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/29169608137e489eb32ff2c6efb9d5b1 2024-12-02T22:14:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7fde6e1692be4aa28c6ccf08934ed5db as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db 2024-12-02T22:14:20,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db, entries=150, sequenceid=52, filesize=30.2 K 2024-12-02T22:14:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/86eb614d17a84ed6ac2dc00f20c2357a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a 2024-12-02T22:14:20,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a, entries=150, sequenceid=52, filesize=11.7 K 2024-12-02T22:14:20,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/29169608137e489eb32ff2c6efb9d5b1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1 2024-12-02T22:14:20,436 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1, entries=150, sequenceid=52, filesize=11.7 K 2024-12-02T22:14:20,442 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for e76fa5b70a6b892d2b9930c0da4f98cd in 1114ms, sequenceid=52, compaction requested=false 2024-12-02T22:14:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-02T22:14:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-02T22:14:20,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-02T22:14:20,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5270 sec 2024-12-02T22:14:20,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.5410 sec 2024-12-02T22:14:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:20,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:20,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:20,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120238eaac49a38e427ab183d780ddf0d514_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:20,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177720609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177720617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177720619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177720628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177720629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741953_1129 (size=12154) 2024-12-02T22:14:20,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:20,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120238eaac49a38e427ab183d780ddf0d514_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120238eaac49a38e427ab183d780ddf0d514_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:20,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e9051e69979b4b96a4cb89c43035bf31, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:20,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e9051e69979b4b96a4cb89c43035bf31 is 175, key is test_row_0/A:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:20,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741954_1130 (size=30955) 2024-12-02T22:14:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177720734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177720734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177720737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177720743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177720743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177720944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177720944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177720953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177720953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177720953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,082 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:14:21,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e9051e69979b4b96a4cb89c43035bf31 2024-12-02T22:14:21,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b8888a2d884e4ec78f46b83cfee72815 is 50, key is test_row_0/B:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741955_1131 (size=12001) 2024-12-02T22:14:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177721263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b8888a2d884e4ec78f46b83cfee72815 2024-12-02T22:14:21,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f5e48befff2a472a8f94608b41b5f774 is 50, key is test_row_0/C:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:21,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741956_1132 (size=12001) 2024-12-02T22:14:21,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177721774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177721774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177721774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177721774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177721775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f5e48befff2a472a8f94608b41b5f774 2024-12-02T22:14:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e9051e69979b4b96a4cb89c43035bf31 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31 2024-12-02T22:14:22,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31, entries=150, sequenceid=67, filesize=30.2 K 2024-12-02T22:14:22,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b8888a2d884e4ec78f46b83cfee72815 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815 2024-12-02T22:14:22,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815, entries=150, sequenceid=67, filesize=11.7 K 2024-12-02T22:14:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f5e48befff2a472a8f94608b41b5f774 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774 2024-12-02T22:14:22,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774, entries=150, sequenceid=67, filesize=11.7 K 2024-12-02T22:14:22,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e76fa5b70a6b892d2b9930c0da4f98cd in 1512ms, sequenceid=67, compaction requested=true 2024-12-02T22:14:22,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:22,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:22,092 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:22,092 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T22:14:22,093 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:22,093 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:22,093 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:22,093 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,094 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=99.1 K 2024-12-02T22:14:22,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:22,094 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,094 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31] 2024-12-02T22:14:22,094 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:22,094 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:22,094 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,095 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.2 K 2024-12-02T22:14:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T22:14:22,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:22,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:22,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:22,101 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf357e0f0474b30beeeb5f6a6db4cdf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733177657935 2024-12-02T22:14:22,101 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b90bceba192941bc94817d00d39ff0da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733177657935 2024-12-02T22:14:22,102 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86eb614d17a84ed6ac2dc00f20c2357a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733177658162 2024-12-02T22:14:22,102 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fde6e1692be4aa28c6ccf08934ed5db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733177658162 2024-12-02T22:14:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T22:14:22,102 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9051e69979b4b96a4cb89c43035bf31, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:22,102 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b8888a2d884e4ec78f46b83cfee72815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:22,114 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:22,117 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:22,118 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/c390177347db4aa28aeca477863c81fd is 50, key is test_row_0/B:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:22,125 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412020089bd8f097d4755875ee3668c2a2c08_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:22,140 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412020089bd8f097d4755875ee3668c2a2c08_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:22,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020089bd8f097d4755875ee3668c2a2c08_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741957_1133 (size=12104) 2024-12-02T22:14:22,154 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/c390177347db4aa28aeca477863c81fd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c390177347db4aa28aeca477863c81fd 2024-12-02T22:14:22,164 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into c390177347db4aa28aeca477863c81fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:22,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:22,164 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177662092; duration=0sec 2024-12-02T22:14:22,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:22,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:22,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:22,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:22,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:22,166 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,166 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.2 K 2024-12-02T22:14:22,167 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 693afdf779cb4f6480a1f11f8708cc1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733177657935 2024-12-02T22:14:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741958_1134 (size=4469) 2024-12-02T22:14:22,175 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 29169608137e489eb32ff2c6efb9d5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733177658162 2024-12-02T22:14:22,175 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e48befff2a472a8f94608b41b5f774, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:22,189 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#114 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:22,189 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/1bec84565f314defa56f2fbe9f7df7cf is 175, key is test_row_0/A:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T22:14:22,215 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:22,216 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/663ccbb6b07c4d8e9f6f6f42d2d31faa is 50, key is test_row_0/C:col10/1733177660579/Put/seqid=0 2024-12-02T22:14:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741959_1135 (size=31058) 2024-12-02T22:14:22,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,256 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741960_1136 (size=12104) 2024-12-02T22:14:22,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3e1d0faef1a4b53a2610d6a67c1670a_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177660626/Put/seqid=0 2024-12-02T22:14:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741961_1137 (size=12154) 2024-12-02T22:14:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T22:14:22,644 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/1bec84565f314defa56f2fbe9f7df7cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf 2024-12-02T22:14:22,651 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 1bec84565f314defa56f2fbe9f7df7cf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:22,652 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:22,652 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177662091; duration=0sec 2024-12-02T22:14:22,652 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:22,652 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:22,696 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/663ccbb6b07c4d8e9f6f6f42d2d31faa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/663ccbb6b07c4d8e9f6f6f42d2d31faa 2024-12-02T22:14:22,705 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 663ccbb6b07c4d8e9f6f6f42d2d31faa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:22,705 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:22,705 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177662092; duration=0sec 2024-12-02T22:14:22,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:22,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T22:14:22,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:22,766 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3e1d0faef1a4b53a2610d6a67c1670a_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3e1d0faef1a4b53a2610d6a67c1670a_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/775280c693ed4066a819c13fbddcee00, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/775280c693ed4066a819c13fbddcee00 is 175, key is test_row_0/A:col10/1733177660626/Put/seqid=0 2024-12-02T22:14:22,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:22,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741962_1138 (size=30955) 2024-12-02T22:14:22,806 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/775280c693ed4066a819c13fbddcee00 2024-12-02T22:14:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/ce77e856eec44f5b9c0c7a3d5ca3d97f is 50, key is test_row_0/B:col10/1733177660626/Put/seqid=0 2024-12-02T22:14:22,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741963_1139 (size=12001) 2024-12-02T22:14:22,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/ce77e856eec44f5b9c0c7a3d5ca3d97f 2024-12-02T22:14:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177722829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177722829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177722842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177722851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177722852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/9fc5062526b245de8522cc674ef9191d is 50, key is test_row_0/C:col10/1733177660626/Put/seqid=0 2024-12-02T22:14:22,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741964_1140 (size=12001) 2024-12-02T22:14:22,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/9fc5062526b245de8522cc674ef9191d 2024-12-02T22:14:22,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/775280c693ed4066a819c13fbddcee00 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00 2024-12-02T22:14:22,927 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00, entries=150, sequenceid=86, filesize=30.2 K 2024-12-02T22:14:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/ce77e856eec44f5b9c0c7a3d5ca3d97f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f 2024-12-02T22:14:22,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T22:14:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/9fc5062526b245de8522cc674ef9191d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d 2024-12-02T22:14:22,941 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T22:14:22,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e76fa5b70a6b892d2b9930c0da4f98cd in 686ms, sequenceid=86, compaction requested=false 2024-12-02T22:14:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T22:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T22:14:22,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T22:14:22,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 844 msec 2024-12-02T22:14:22,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 855 msec 2024-12-02T22:14:22,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:22,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021c9253b02daf4776ac90396f89756020_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177722989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177722992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177722999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177723001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177723005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741965_1141 (size=14594) 2024-12-02T22:14:23,051 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:23,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021c9253b02daf4776ac90396f89756020_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c9253b02daf4776ac90396f89756020_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:23,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0b0bb450ea114ee4ab3300bc5e758b62, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0b0bb450ea114ee4ab3300bc5e758b62 is 175, key is test_row_0/A:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741966_1142 (size=39549) 2024-12-02T22:14:23,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0b0bb450ea114ee4ab3300bc5e758b62 2024-12-02T22:14:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177723113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177723113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177723113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177723113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177723124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/09c6a34613224bfc94f0b0aed7ca4ef9 is 50, key is test_row_0/B:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741967_1143 (size=12001) 2024-12-02T22:14:23,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/09c6a34613224bfc94f0b0aed7ca4ef9 2024-12-02T22:14:23,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f485a5439d47439bb9b63430b1c3635f is 50, key is test_row_0/C:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741968_1144 (size=12001) 2024-12-02T22:14:23,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f485a5439d47439bb9b63430b1c3635f 2024-12-02T22:14:23,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0b0bb450ea114ee4ab3300bc5e758b62 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62 2024-12-02T22:14:23,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62, entries=200, sequenceid=104, filesize=38.6 K 2024-12-02T22:14:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/09c6a34613224bfc94f0b0aed7ca4ef9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9 2024-12-02T22:14:23,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T22:14:23,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f485a5439d47439bb9b63430b1c3635f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f 2024-12-02T22:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T22:14:23,253 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:23,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T22:14:23,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:23,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e76fa5b70a6b892d2b9930c0da4f98cd in 296ms, sequenceid=104, compaction requested=true 2024-12-02T22:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T22:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:23,257 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:23,258 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:23,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:23,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:23,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:23,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:23,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:23,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:23,260 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,260 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=99.2 K 2024-12-02T22:14:23,260 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62] 2024-12-02T22:14:23,261 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bec84565f314defa56f2fbe9f7df7cf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:23,262 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775280c693ed4066a819c13fbddcee00, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733177660614 2024-12-02T22:14:23,264 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0bb450ea114ee4ab3300bc5e758b62, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:23,279 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,279 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:23,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:23,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:23,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:23,283 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:23,283 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:23,283 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,283 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c390177347db4aa28aeca477863c81fd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.3 K 2024-12-02T22:14:23,284 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c390177347db4aa28aeca477863c81fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:23,285 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ce77e856eec44f5b9c0c7a3d5ca3d97f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733177660614 2024-12-02T22:14:23,285 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c6a34613224bfc94f0b0aed7ca4ef9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:23,287 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202aa0c6e6d98ca473dbe8f64d5276446e8_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,290 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202aa0c6e6d98ca473dbe8f64d5276446e8_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,290 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202aa0c6e6d98ca473dbe8f64d5276446e8_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,305 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:23,306 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/6fa2ed96799e4dbf92da9f0f063eeaa4 is 50, key is test_row_0/B:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741969_1145 (size=4469) 2024-12-02T22:14:23,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741970_1146 (size=12207) 2024-12-02T22:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:23,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:14:23,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177662999, max timestamp=1733177663341Num uniques 859; 2024-12-02T22:14:23,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:23,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177662999, max timestamp=1733177663341Num uniques 859; 2024-12-02T22:14:23,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:23,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177662999, max timestamp=1733177663341Num uniques 859; 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:23,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:23,359 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/6fa2ed96799e4dbf92da9f0f063eeaa4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/6fa2ed96799e4dbf92da9f0f063eeaa4 2024-12-02T22:14:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:23,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202faba7a0feb5448d79f56b6fdfa86e654_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177662999/Put/seqid=0 2024-12-02T22:14:23,385 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into 6fa2ed96799e4dbf92da9f0f063eeaa4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:23,386 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:23,386 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177663279; duration=0sec 2024-12-02T22:14:23,386 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:23,386 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:23,386 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:23,389 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:23,389 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:23,389 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,389 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/663ccbb6b07c4d8e9f6f6f42d2d31faa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.3 K 2024-12-02T22:14:23,390 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 663ccbb6b07c4d8e9f6f6f42d2d31faa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733177660577 2024-12-02T22:14:23,391 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc5062526b245de8522cc674ef9191d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733177660614 2024-12-02T22:14:23,391 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f485a5439d47439bb9b63430b1c3635f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:23,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733177663382, max timestamp=1733177663402Num uniques 859; 2024-12-02T22:14:23,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:23,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733177663382, max timestamp=1733177663402Num uniques 859; 2024-12-02T22:14:23,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177663382, max timestamp=1733177663402Num uniques 859; 2024-12-02T22:14:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:14:23,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741971_1147 (size=12154) 2024-12-02T22:14:23,425 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:23,426 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/097c369ef8ce4391ab3e9f309600d627 is 50, key is test_row_0/C:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:23,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202faba7a0feb5448d79f56b6fdfa86e654_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202faba7a0feb5448d79f56b6fdfa86e654_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:23,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7132cd6abe34487ba6dd4121f9ee6bf2, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:23,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7132cd6abe34487ba6dd4121f9ee6bf2 is 175, key is test_row_0/A:col10/1733177662999/Put/seqid=0 2024-12-02T22:14:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741972_1148 (size=12207) 2024-12-02T22:14:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733177663421, max timestamp=1733177663459Num uniques 1118; 2024-12-02T22:14:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733177663421, max timestamp=1733177663460Num uniques 1118; 2024-12-02T22:14:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T22:14:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733177663421, max timestamp=1733177663460Num uniques 1118; 2024-12-02T22:14:23,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T22:14:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T22:14:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741973_1149 (size=30955) 2024-12-02T22:14:23,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7132cd6abe34487ba6dd4121f9ee6bf2 2024-12-02T22:14:23,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/65d8b953dc584d8985ec25d013c60067 is 50, key is test_row_0/B:col10/1733177662999/Put/seqid=0 2024-12-02T22:14:23,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741974_1150 (size=12001) 2024-12-02T22:14:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177723525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733177663511, max timestamp=1733177663540Num uniques 1177; 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733177663511, max timestamp=1733177663540Num uniques 1177; 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733177663511, max timestamp=1733177663540Num uniques 1177; 2024-12-02T22:14:23,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T22:14:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:23,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:23,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177663587, max timestamp=1733177663603Num uniques 1136; 2024-12-02T22:14:23,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-02T22:14:23,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177663587, max timestamp=1733177663603Num uniques 1136; 2024-12-02T22:14:23,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-02T22:14:23,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177663587, max timestamp=1733177663603Num uniques 1136; 2024-12-02T22:14:23,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-02T22:14:23,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733177663632, max timestamp=1733177663656Num uniques 1145; 2024-12-02T22:14:23,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-02T22:14:23,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733177663632, max timestamp=1733177663656Num uniques 1145; 2024-12-02T22:14:23,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-02T22:14:23,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733177663632, max timestamp=1733177663656Num uniques 1145; 2024-12-02T22:14:23,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-02T22:14:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177723692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177723691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177723709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177723710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177723710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,721 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#123 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:23,722 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/176a543492054680ac3f75d1eb7b36b9 is 175, key is test_row_0/A:col10/1733177662841/Put/seqid=0 2024-12-02T22:14:23,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741975_1151 (size=31161) 2024-12-02T22:14:23,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177723813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177723813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177723823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177723823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177723823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:23,868 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/097c369ef8ce4391ab3e9f309600d627 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/097c369ef8ce4391ab3e9f309600d627 2024-12-02T22:14:23,877 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 097c369ef8ce4391ab3e9f309600d627(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:23,877 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:23,877 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177663281; duration=0sec 2024-12-02T22:14:23,877 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:23,877 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:23,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:23,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:23,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/65d8b953dc584d8985ec25d013c60067 2024-12-02T22:14:23,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/98e424e395e446c39aa68b1ba88be61d is 50, key is test_row_0/C:col10/1733177662999/Put/seqid=0 2024-12-02T22:14:23,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741976_1152 (size=12001) 2024-12-02T22:14:23,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/98e424e395e446c39aa68b1ba88be61d 2024-12-02T22:14:23,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/7132cd6abe34487ba6dd4121f9ee6bf2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2 2024-12-02T22:14:24,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2, entries=150, sequenceid=125, filesize=30.2 K 2024-12-02T22:14:24,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/65d8b953dc584d8985ec25d013c60067 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067 2024-12-02T22:14:24,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T22:14:24,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/98e424e395e446c39aa68b1ba88be61d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d 2024-12-02T22:14:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177724023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T22:14:24,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=167.72 KB/171750 for e76fa5b70a6b892d2b9930c0da4f98cd in 684ms, sequenceid=125, compaction requested=false 2024-12-02T22:14:24,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:24,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733177663689, max timestamp=1733177664028Num uniques 1254; 2024-12-02T22:14:24,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T22:14:24,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:24,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=125.23 KB heapSize=344.30 KB 2024-12-02T22:14:24,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733177663689, max timestamp=1733177664028Num uniques 1254; 2024-12-02T22:14:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:24,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733177663689, max timestamp=1733177664028Num uniques 1254; 2024-12-02T22:14:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T22:14:24,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-02T22:14:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:24,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202617d8c0656854954bec9dbef45150075_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741977_1153 (size=19774) 2024-12-02T22:14:24,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:24,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202617d8c0656854954bec9dbef45150075_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202617d8c0656854954bec9dbef45150075_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:24,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/717a162a24254c388773b76043fcb1b9, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:24,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/717a162a24254c388773b76043fcb1b9 is 175, key is test_row_0/A:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741978_1154 (size=57033) 2024-12-02T22:14:24,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733177664116, max timestamp=1733177664139Num uniques 859; 2024-12-02T22:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T22:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733177664116, max timestamp=1733177664139Num uniques 859; 2024-12-02T22:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T22:14:24,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177664121, max timestamp=1733177664139Num uniques 859; 2024-12-02T22:14:24,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:14:24,170 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/176a543492054680ac3f75d1eb7b36b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9 2024-12-02T22:14:24,179 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 176a543492054680ac3f75d1eb7b36b9(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:24,179 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:24,179 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177663258; duration=0sec 2024-12-02T22:14:24,179 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:24,179 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:24,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177724187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177724189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177724210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177724210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177724313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177724313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177724323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177724333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177724333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:24,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/717a162a24254c388773b76043fcb1b9 2024-12-02T22:14:24,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:24,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9c5eef5fb68c49cd963865e8bb6c7a6f is 50, key is test_row_0/B:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741979_1155 (size=12151) 2024-12-02T22:14:24,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9c5eef5fb68c49cd963865e8bb6c7a6f 2024-12-02T22:14:24,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/bd091215ef014373a330f8280eac8508 is 50, key is test_row_0/C:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177724533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177724533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177724533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741980_1156 (size=12151) 2024-12-02T22:14:24,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/bd091215ef014373a330f8280eac8508 2024-12-02T22:14:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177724543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/717a162a24254c388773b76043fcb1b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9 2024-12-02T22:14:24,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9, entries=300, sequenceid=236, filesize=55.7 K 2024-12-02T22:14:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9c5eef5fb68c49cd963865e8bb6c7a6f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f 2024-12-02T22:14:24,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f, entries=150, sequenceid=236, filesize=11.9 K 2024-12-02T22:14:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/bd091215ef014373a330f8280eac8508 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508 2024-12-02T22:14:24,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508, entries=150, sequenceid=236, filesize=11.9 K 2024-12-02T22:14:24,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.50 KB/109056, currentSize=147.60 KB/151140 for e76fa5b70a6b892d2b9930c0da4f98cd in 552ms, sequenceid=236, compaction requested=true 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:24,587 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:24,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:24,588 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:24,589 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:24,589 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:24,589 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,589 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=116.4 K 2024-12-02T22:14:24,589 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,590 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9] 2024-12-02T22:14:24,590 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:24,590 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:24,590 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,590 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/6fa2ed96799e4dbf92da9f0f063eeaa4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.5 K 2024-12-02T22:14:24,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa2ed96799e4dbf92da9f0f063eeaa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:24,591 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176a543492054680ac3f75d1eb7b36b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:24,592 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d8b953dc584d8985ec25d013c60067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733177662999 2024-12-02T22:14:24,592 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7132cd6abe34487ba6dd4121f9ee6bf2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733177662999 2024-12-02T22:14:24,592 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5eef5fb68c49cd963865e8bb6c7a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663706 2024-12-02T22:14:24,592 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717a162a24254c388773b76043fcb1b9, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663689 2024-12-02T22:14:24,617 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:24,617 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e2d72bb91ff44773bfcd590197c1cc2e is 50, key is test_row_0/B:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,621 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:24,637 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412021521f399ada0498aa99f056ad90d8eae_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:24,639 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412021521f399ada0498aa99f056ad90d8eae_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:24,640 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021521f399ada0498aa99f056ad90d8eae_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:24,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T22:14:24,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:24,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741981_1157 (size=12459) 2024-12-02T22:14:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025b963f8939aa43b0a680ef92d3153ab0_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177664185/Put/seqid=0 2024-12-02T22:14:24,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741982_1158 (size=4469) 2024-12-02T22:14:24,727 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#133 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:24,728 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bb66f2e4b2f4880ab7da8a23e4c63c9 is 175, key is test_row_0/A:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741983_1159 (size=12454) 2024-12-02T22:14:24,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741984_1160 (size=31413) 2024-12-02T22:14:24,762 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bb66f2e4b2f4880ab7da8a23e4c63c9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9 2024-12-02T22:14:24,773 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 8bb66f2e4b2f4880ab7da8a23e4c63c9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:24,773 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:24,773 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177664587; duration=0sec 2024-12-02T22:14:24,773 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:24,773 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:24,773 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:24,775 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:24,775 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:24,775 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:24,775 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/097c369ef8ce4391ab3e9f309600d627, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=35.5 K 2024-12-02T22:14:24,776 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 097c369ef8ce4391ab3e9f309600d627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733177662823 2024-12-02T22:14:24,776 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e424e395e446c39aa68b1ba88be61d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733177662999 2024-12-02T22:14:24,778 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd091215ef014373a330f8280eac8508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663706 2024-12-02T22:14:24,787 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:24,787 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/de8a24e8064c40a99fef3524bf2f2a6e is 50, key is test_row_0/C:col10/1733177664040/Put/seqid=0 2024-12-02T22:14:24,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741985_1161 (size=12459) 2024-12-02T22:14:24,805 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/de8a24e8064c40a99fef3524bf2f2a6e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/de8a24e8064c40a99fef3524bf2f2a6e 2024-12-02T22:14:24,810 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into de8a24e8064c40a99fef3524bf2f2a6e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:24,810 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:24,810 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177664587; duration=0sec 2024-12-02T22:14:24,810 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:24,810 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:24,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177724853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177724854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177724856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177724857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177724858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177724973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177724973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177724973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177724973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,085 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e2d72bb91ff44773bfcd590197c1cc2e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e2d72bb91ff44773bfcd590197c1cc2e 2024-12-02T22:14:25,092 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into e2d72bb91ff44773bfcd590197c1cc2e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:25,092 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:25,092 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177664587; duration=0sec 2024-12-02T22:14:25,093 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:25,093 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:25,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:25,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025b963f8939aa43b0a680ef92d3153ab0_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b963f8939aa43b0a680ef92d3153ab0_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:25,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bc8b85e62a842c1985d26d152d86900, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bc8b85e62a842c1985d26d152d86900 is 175, key is test_row_0/A:col10/1733177664185/Put/seqid=0 2024-12-02T22:14:25,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741986_1162 (size=31255) 2024-12-02T22:14:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177725183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177725183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177725183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177725184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177725373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:25,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177725503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177725503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177725503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177725503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:25,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bc8b85e62a842c1985d26d152d86900 2024-12-02T22:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a90fa6675d0b463180d4208b2aa6de6c is 50, key is test_row_0/B:col10/1733177664185/Put/seqid=0 2024-12-02T22:14:25,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741987_1163 (size=12301) 2024-12-02T22:14:25,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a90fa6675d0b463180d4208b2aa6de6c 2024-12-02T22:14:25,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/46dd99d0c9d64ba385f2e2b2372db52c is 50, key is test_row_0/C:col10/1733177664185/Put/seqid=0 2024-12-02T22:14:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741988_1164 (size=12301) 2024-12-02T22:14:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177726013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:26,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177726013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177726013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177726013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:26,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/46dd99d0c9d64ba385f2e2b2372db52c 2024-12-02T22:14:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177726383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:26,388 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in e76fa5b70a6b892d2b9930c0da4f98cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/8bc8b85e62a842c1985d26d152d86900 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900 2024-12-02T22:14:26,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900, entries=150, sequenceid=273, filesize=30.5 K 2024-12-02T22:14:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a90fa6675d0b463180d4208b2aa6de6c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c 2024-12-02T22:14:26,406 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T22:14:26,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/46dd99d0c9d64ba385f2e2b2372db52c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c 2024-12-02T22:14:26,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c, entries=150, sequenceid=273, filesize=12.0 K 2024-12-02T22:14:26,414 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for e76fa5b70a6b892d2b9930c0da4f98cd in 1757ms, sequenceid=273, compaction requested=false 2024-12-02T22:14:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T22:14:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T22:14:26,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T22:14:26,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1580 sec 2024-12-02T22:14:26,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.1650 sec 2024-12-02T22:14:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:27,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:27,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202084a46585db6416982f922e513d22d55_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741989_1165 (size=12454) 2024-12-02T22:14:27,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:27,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202084a46585db6416982f922e513d22d55_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202084a46585db6416982f922e513d22d55_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:27,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3bbd9fe8d8b84170ad3f6bdea046f8b7, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:27,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 is 175, key is test_row_0/A:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177667048, max timestamp=1733177667060Num uniques 859; 2024-12-02T22:14:27,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:14:27,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177667048, max timestamp=1733177667060Num uniques 859; 2024-12-02T22:14:27,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:14:27,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177667048, max timestamp=1733177667060Num uniques 859; 2024-12-02T22:14:27,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:14:27,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741990_1166 (size=31255) 2024-12-02T22:14:27,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 2024-12-02T22:14:27,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/4e7922609221495da0c21afef1ff8047 is 50, key is test_row_0/B:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741991_1167 (size=12301) 2024-12-02T22:14:27,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/4e7922609221495da0c21afef1ff8047 2024-12-02T22:14:27,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/55dd91df9cbc4ac48da3dcfabe9a5f18 is 50, key is test_row_0/C:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741992_1168 (size=12301) 2024-12-02T22:14:27,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177727122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177727122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177727122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177727124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177727233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177727233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177727233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177727233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T22:14:27,413 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:27,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T22:14:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:27,416 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:27,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:27,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177727443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177727443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177727444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177727444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/55dd91df9cbc4ac48da3dcfabe9a5f18 2024-12-02T22:14:27,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 2024-12-02T22:14:27,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7, entries=150, sequenceid=286, filesize=30.5 K 2024-12-02T22:14:27,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/4e7922609221495da0c21afef1ff8047 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047 2024-12-02T22:14:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:27,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T22:14:27,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/55dd91df9cbc4ac48da3dcfabe9a5f18 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18 2024-12-02T22:14:27,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T22:14:27,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e76fa5b70a6b892d2b9930c0da4f98cd in 507ms, sequenceid=286, compaction requested=true 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:27,532 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:27,532 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:27,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:27,534 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:27,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:27,534 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:27,534 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:27,534 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:27,534 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=91.7 K 2024-12-02T22:14:27,534 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e2d72bb91ff44773bfcd590197c1cc2e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.2 K 2024-12-02T22:14:27,534 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:27,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7] 2024-12-02T22:14:27,535 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb66f2e4b2f4880ab7da8a23e4c63c9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663706 2024-12-02T22:14:27,535 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d72bb91ff44773bfcd590197c1cc2e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663706 2024-12-02T22:14:27,536 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a90fa6675d0b463180d4208b2aa6de6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733177664185 2024-12-02T22:14:27,536 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc8b85e62a842c1985d26d152d86900, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733177664185 2024-12-02T22:14:27,537 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7922609221495da0c21afef1ff8047, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:27,537 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbd9fe8d8b84170ad3f6bdea046f8b7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:27,545 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:27,546 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/4d7bbc831e3243fa922eb86cfb7c0ef3 is 50, key is test_row_0/B:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,546 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:27,552 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ad65856815004c50960197c0128a84c2_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:27,554 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ad65856815004c50960197c0128a84c2_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:27,554 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ad65856815004c50960197c0128a84c2_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741993_1169 (size=12711) 2024-12-02T22:14:27,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T22:14:27,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:27,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:27,573 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/4d7bbc831e3243fa922eb86cfb7c0ef3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4d7bbc831e3243fa922eb86cfb7c0ef3 2024-12-02T22:14:27,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741994_1170 (size=4469) 2024-12-02T22:14:27,585 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into 4d7bbc831e3243fa922eb86cfb7c0ef3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:27,585 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:27,585 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177667532; duration=0sec 2024-12-02T22:14:27,585 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:27,585 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:27,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:27,587 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:27,587 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:27,587 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:27,587 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/de8a24e8064c40a99fef3524bf2f2a6e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.2 K 2024-12-02T22:14:27,588 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting de8a24e8064c40a99fef3524bf2f2a6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733177663706 2024-12-02T22:14:27,588 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 46dd99d0c9d64ba385f2e2b2372db52c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733177664185 2024-12-02T22:14:27,588 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 55dd91df9cbc4ac48da3dcfabe9a5f18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:27,600 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:27,601 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/e54047f59bf045e5b07bebda07f86b34 is 50, key is test_row_0/C:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020af52c8455174282a1c5a8a601c6b2f9_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177667096/Put/seqid=0 2024-12-02T22:14:27,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741995_1171 (size=12454) 2024-12-02T22:14:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741996_1172 (size=12711) 2024-12-02T22:14:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:27,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177727770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177727770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177727771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177727772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177727883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177727883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177727884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177727884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:27,980 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:27,981 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0aa871a0c121422fba860d1e284bf4db is 175, key is test_row_0/A:col10/1733177664855/Put/seqid=0 2024-12-02T22:14:27,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741997_1173 (size=31665) 2024-12-02T22:14:27,994 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/0aa871a0c121422fba860d1e284bf4db as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db 2024-12-02T22:14:28,000 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 0aa871a0c121422fba860d1e284bf4db(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:28,000 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:28,000 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177667532; duration=0sec 2024-12-02T22:14:28,000 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:28,000 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:28,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:28,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020af52c8455174282a1c5a8a601c6b2f9_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020af52c8455174282a1c5a8a601c6b2f9_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/040bc51f1e6440c691b0ef1617264211, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/040bc51f1e6440c691b0ef1617264211 is 175, key is test_row_0/A:col10/1733177667096/Put/seqid=0 2024-12-02T22:14:28,037 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/e54047f59bf045e5b07bebda07f86b34 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e54047f59bf045e5b07bebda07f86b34 2024-12-02T22:14:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741998_1174 (size=31255) 2024-12-02T22:14:28,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/040bc51f1e6440c691b0ef1617264211 2024-12-02T22:14:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:28,044 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into e54047f59bf045e5b07bebda07f86b34(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:28,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:28,044 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177667532; duration=0sec 2024-12-02T22:14:28,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:28,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:28,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/949f524f7b7e442c8d9581c528937af5 is 50, key is test_row_0/B:col10/1733177667096/Put/seqid=0 2024-12-02T22:14:28,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741999_1175 (size=12301) 2024-12-02T22:14:28,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177728093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177728094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177728094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177728094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177728403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177728403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177728403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177728404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177728413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,417 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in e76fa5b70a6b892d2b9930c0da4f98cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,457 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/949f524f7b7e442c8d9581c528937af5 2024-12-02T22:14:28,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/935726590ed84e10880c10a54a4b4d8b is 50, key is test_row_0/C:col10/1733177667096/Put/seqid=0 2024-12-02T22:14:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742000_1176 (size=12301) 2024-12-02T22:14:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:28,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/935726590ed84e10880c10a54a4b4d8b 2024-12-02T22:14:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/040bc51f1e6440c691b0ef1617264211 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211 2024-12-02T22:14:28,882 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211, entries=150, sequenceid=322, filesize=30.5 K 2024-12-02T22:14:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/949f524f7b7e442c8d9581c528937af5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5 2024-12-02T22:14:28,887 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5, entries=150, sequenceid=322, filesize=12.0 K 2024-12-02T22:14:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/935726590ed84e10880c10a54a4b4d8b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b 2024-12-02T22:14:28,893 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b, entries=150, sequenceid=322, filesize=12.0 K 2024-12-02T22:14:28,894 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e76fa5b70a6b892d2b9930c0da4f98cd in 1325ms, sequenceid=322, compaction requested=false 2024-12-02T22:14:28,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:28,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:28,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T22:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T22:14:28,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T22:14:28,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-12-02T22:14:28,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4840 sec 2024-12-02T22:14:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:28,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:28,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:28,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f0adb1e74304448840d3ed1b00b3964_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:28,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742001_1177 (size=14994) 2024-12-02T22:14:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177728950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177728953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177728955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177728955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177729064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177729064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177729064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177729073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,230 INFO [master/7ae73638d47e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T22:14:29,230 INFO [master/7ae73638d47e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T22:14:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177729273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177729273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177729273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177729283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:29,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f0adb1e74304448840d3ed1b00b3964_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f0adb1e74304448840d3ed1b00b3964_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:29,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ed8d0609463843c0b950104b8c608292, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:29,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ed8d0609463843c0b950104b8c608292 is 175, key is test_row_0/A:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:29,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742002_1178 (size=39949) 2024-12-02T22:14:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T22:14:29,564 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:29,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T22:14:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:29,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:29,567 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:29,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177729583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177729583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177729583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177729593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:29,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T22:14:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:29,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ed8d0609463843c0b950104b8c608292 2024-12-02T22:14:29,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e0e13fda6bae4bbbad6d404cba79dc61 is 50, key is test_row_0/B:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:29,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742003_1179 (size=12301) 2024-12-02T22:14:29,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e0e13fda6bae4bbbad6d404cba79dc61 2024-12-02T22:14:29,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/7aa3b4cc02a24e52b1233bc88bede101 is 50, key is test_row_0/C:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:29,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742004_1180 (size=12301) 2024-12-02T22:14:29,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T22:14:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:29,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:30,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T22:14:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177730093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177730093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177730093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:30,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177730103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:30,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T22:14:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:30,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/7aa3b4cc02a24e52b1233bc88bede101 2024-12-02T22:14:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:30,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ed8d0609463843c0b950104b8c608292 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292 2024-12-02T22:14:30,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292, entries=200, sequenceid=337, filesize=39.0 K 2024-12-02T22:14:30,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e0e13fda6bae4bbbad6d404cba79dc61 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61 2024-12-02T22:14:30,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T22:14:30,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/7aa3b4cc02a24e52b1233bc88bede101 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101 2024-12-02T22:14:30,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T22:14:30,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e76fa5b70a6b892d2b9930c0da4f98cd in 1297ms, sequenceid=337, compaction requested=true 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:30,216 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:30,216 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:30,217 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:30,217 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:30,217 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:30,217 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:30,217 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,217 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,217 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=100.5 K 2024-12-02T22:14:30,218 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4d7bbc831e3243fa922eb86cfb7c0ef3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.4 K 2024-12-02T22:14:30,218 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,218 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292] 2024-12-02T22:14:30,218 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d7bbc831e3243fa922eb86cfb7c0ef3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:30,218 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa871a0c121422fba860d1e284bf4db, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:30,218 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 949f524f7b7e442c8d9581c528937af5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733177667088 2024-12-02T22:14:30,218 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040bc51f1e6440c691b0ef1617264211, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733177667088 2024-12-02T22:14:30,219 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8d0609463843c0b950104b8c608292, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667768 2024-12-02T22:14:30,219 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e13fda6bae4bbbad6d404cba79dc61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667770 2024-12-02T22:14:30,228 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:30,229 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:30,230 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/59a38d540fad48949dbd2b81511d43cd is 50, key is test_row_0/B:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:30,231 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d3559a41057a4b87abd576618a190c74_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:30,233 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d3559a41057a4b87abd576618a190c74_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:30,233 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d3559a41057a4b87abd576618a190c74_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:30,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742005_1181 (size=12813) 2024-12-02T22:14:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742006_1182 (size=4469) 2024-12-02T22:14:30,244 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/59a38d540fad48949dbd2b81511d43cd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/59a38d540fad48949dbd2b81511d43cd 2024-12-02T22:14:30,250 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into 59a38d540fad48949dbd2b81511d43cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:30,250 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:30,250 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177670216; duration=0sec 2024-12-02T22:14:30,250 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:30,250 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:30,250 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:30,251 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:30,251 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:30,251 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,252 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e54047f59bf045e5b07bebda07f86b34, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.4 K 2024-12-02T22:14:30,252 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e54047f59bf045e5b07bebda07f86b34, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177664855 2024-12-02T22:14:30,252 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 935726590ed84e10880c10a54a4b4d8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733177667088 2024-12-02T22:14:30,253 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa3b4cc02a24e52b1233bc88bede101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667770 2024-12-02T22:14:30,261 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:30,261 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/40b4a777eedd46d4b2d38b3d23930663 is 50, key is test_row_0/C:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742007_1183 (size=12813) 2024-12-02T22:14:30,280 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/40b4a777eedd46d4b2d38b3d23930663 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/40b4a777eedd46d4b2d38b3d23930663 2024-12-02T22:14:30,285 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 40b4a777eedd46d4b2d38b3d23930663(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:30,285 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:30,285 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177670216; duration=0sec 2024-12-02T22:14:30,287 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:30,287 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:30,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:30,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:30,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120289f26509f22b4a3d8fc20adae593ca62_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177668943/Put/seqid=0 2024-12-02T22:14:30,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742008_1184 (size=12454) 2024-12-02T22:14:30,646 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:30,647 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/06103b7188e842868d9c3a9408d338fa is 175, key is test_row_0/A:col10/1733177668917/Put/seqid=0 2024-12-02T22:14:30,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742009_1185 (size=31767) 2024-12-02T22:14:30,663 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/06103b7188e842868d9c3a9408d338fa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa 2024-12-02T22:14:30,671 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 06103b7188e842868d9c3a9408d338fa(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:30,671 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:30,671 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177670216; duration=0sec 2024-12-02T22:14:30,671 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:30,671 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:30,753 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120289f26509f22b4a3d8fc20adae593ca62_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120289f26509f22b4a3d8fc20adae593ca62_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5ae8d84929a04ef596597cac429af513, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5ae8d84929a04ef596597cac429af513 is 175, key is test_row_0/A:col10/1733177668943/Put/seqid=0 2024-12-02T22:14:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742010_1186 (size=31255) 2024-12-02T22:14:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:31,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:31,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177731119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177731119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177731121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177731121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,171 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5ae8d84929a04ef596597cac429af513 2024-12-02T22:14:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/3da5322808a54a1ba9adec931128040b is 50, key is test_row_0/B:col10/1733177668943/Put/seqid=0 2024-12-02T22:14:31,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742011_1187 (size=12301) 2024-12-02T22:14:31,187 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/3da5322808a54a1ba9adec931128040b 2024-12-02T22:14:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/42584cb9459544e69646c3df34611778 is 50, key is test_row_0/C:col10/1733177668943/Put/seqid=0 2024-12-02T22:14:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742012_1188 (size=12301) 2024-12-02T22:14:31,209 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/42584cb9459544e69646c3df34611778 2024-12-02T22:14:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5ae8d84929a04ef596597cac429af513 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513 2024-12-02T22:14:31,224 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513, entries=150, sequenceid=359, filesize=30.5 K 2024-12-02T22:14:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/3da5322808a54a1ba9adec931128040b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b 2024-12-02T22:14:31,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T22:14:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/42584cb9459544e69646c3df34611778 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778 2024-12-02T22:14:31,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177731233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177731233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177731234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177731236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,239 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T22:14:31,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e76fa5b70a6b892d2b9930c0da4f98cd in 907ms, sequenceid=359, compaction requested=false 2024-12-02T22:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T22:14:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T22:14:31,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T22:14:31,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6740 sec 2024-12-02T22:14:31,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6800 sec 2024-12-02T22:14:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:31,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:31,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:31,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120278a06c0f14de47ddad61761c9d771714_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:31,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742013_1189 (size=17534) 2024-12-02T22:14:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177731467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177731469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177731470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177731470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177731583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177731583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177731583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177731584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T22:14:31,714 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:31,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T22:14:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:31,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:31,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:31,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:31,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177731793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177731793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177731793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177731794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:31,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:31,863 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120278a06c0f14de47ddad61761c9d771714_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120278a06c0f14de47ddad61761c9d771714_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:31,864 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/08817679927840b8be447e1b55520172, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:31,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/08817679927840b8be447e1b55520172 is 175, key is test_row_0/A:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:31,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742014_1190 (size=48639) 2024-12-02T22:14:31,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:31,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177732103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177732103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177732103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177732104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/08817679927840b8be447e1b55520172 2024-12-02T22:14:32,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a7c891129abe4657baf8ce22820de5cf is 50, key is test_row_0/B:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742015_1191 (size=12301) 2024-12-02T22:14:32,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47976 deadline: 1733177732423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,429 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in e76fa5b70a6b892d2b9930c0da4f98cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177732613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177732614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177732614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177732614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:32,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a7c891129abe4657baf8ce22820de5cf 2024-12-02T22:14:32,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/64407bdf34554a8b87e0cb712f905a6d is 50, key is test_row_0/C:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:32,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742016_1192 (size=12301) 2024-12-02T22:14:32,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:32,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:32,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:33,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:33,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:33,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:33,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:33,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/64407bdf34554a8b87e0cb712f905a6d 2024-12-02T22:14:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/08817679927840b8be447e1b55520172 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172 2024-12-02T22:14:33,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172, entries=250, sequenceid=375, filesize=47.5 K 2024-12-02T22:14:33,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/a7c891129abe4657baf8ce22820de5cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf 2024-12-02T22:14:33,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf, entries=150, sequenceid=375, filesize=12.0 K 2024-12-02T22:14:33,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/64407bdf34554a8b87e0cb712f905a6d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d 2024-12-02T22:14:33,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d, entries=150, sequenceid=375, filesize=12.0 K 2024-12-02T22:14:33,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e76fa5b70a6b892d2b9930c0da4f98cd in 1677ms, sequenceid=375, compaction requested=true 2024-12-02T22:14:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:33,122 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:33,122 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:33,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:33,123 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:33,123 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:33,123 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,124 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:33,124 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=109.0 K 2024-12-02T22:14:33,124 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:33,124 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,124 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,124 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172] 2024-12-02T22:14:33,124 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/59a38d540fad48949dbd2b81511d43cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.5 K 2024-12-02T22:14:33,124 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06103b7188e842868d9c3a9408d338fa, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667770 2024-12-02T22:14:33,124 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a38d540fad48949dbd2b81511d43cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667770 2024-12-02T22:14:33,125 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae8d84929a04ef596597cac429af513, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733177668943 2024-12-02T22:14:33,125 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da5322808a54a1ba9adec931128040b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733177668943 2024-12-02T22:14:33,125 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08817679927840b8be447e1b55520172, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:33,125 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c891129abe4657baf8ce22820de5cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:33,132 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:33,136 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:33,136 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b352231ce9e94936905ec6a14b25d412 is 50, key is test_row_0/B:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:33,138 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412029bdfaf5b68a54017bf486e344785c433_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:33,139 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412029bdfaf5b68a54017bf486e344785c433_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:33,139 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029bdfaf5b68a54017bf486e344785c433_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742017_1193 (size=12915) 2024-12-02T22:14:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742018_1194 (size=4469) 2024-12-02T22:14:33,156 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b352231ce9e94936905ec6a14b25d412 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b352231ce9e94936905ec6a14b25d412 2024-12-02T22:14:33,157 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#159 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:33,158 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/c45918b134d2445a9c0d74ca6807613b is 175, key is test_row_0/A:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:33,164 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into b352231ce9e94936905ec6a14b25d412(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:33,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:33,164 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177673122; duration=0sec 2024-12-02T22:14:33,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:33,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:33,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:33,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:33,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:33,165 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,165 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/40b4a777eedd46d4b2d38b3d23930663, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.5 K 2024-12-02T22:14:33,166 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b4a777eedd46d4b2d38b3d23930663, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733177667770 2024-12-02T22:14:33,166 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 42584cb9459544e69646c3df34611778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733177668943 2024-12-02T22:14:33,167 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 64407bdf34554a8b87e0cb712f905a6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742019_1195 (size=31869) 2024-12-02T22:14:33,176 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/c45918b134d2445a9c0d74ca6807613b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b 2024-12-02T22:14:33,179 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:33,180 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/7650417eb4824bed8b3797d85ba12e6b is 50, key is test_row_0/C:col10/1733177671443/Put/seqid=0 2024-12-02T22:14:33,188 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into c45918b134d2445a9c0d74ca6807613b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:33,188 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:33,188 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177673122; duration=0sec 2024-12-02T22:14:33,188 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:33,188 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:33,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742020_1196 (size=12915) 2024-12-02T22:14:33,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T22:14:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,249 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c2aee3f3a8384b5abd90c7ddfbf3febf_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177671467/Put/seqid=0 2024-12-02T22:14:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742021_1197 (size=12454) 2024-12-02T22:14:33,600 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/7650417eb4824bed8b3797d85ba12e6b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7650417eb4824bed8b3797d85ba12e6b 2024-12-02T22:14:33,606 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 7650417eb4824bed8b3797d85ba12e6b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:33,606 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:33,606 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177673123; duration=0sec 2024-12-02T22:14:33,606 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:33,606 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177733644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177733645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177733651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177733652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:33,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c2aee3f3a8384b5abd90c7ddfbf3febf_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c2aee3f3a8384b5abd90c7ddfbf3febf_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:33,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/152481d4f4d6441b9c70e8c222e5177a, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/152481d4f4d6441b9c70e8c222e5177a is 175, key is test_row_0/A:col10/1733177671467/Put/seqid=0 2024-12-02T22:14:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742022_1198 (size=31255) 2024-12-02T22:14:33,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/152481d4f4d6441b9c70e8c222e5177a 2024-12-02T22:14:33,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/7f8276004ad04222a6aebbc4f5d8f96e is 50, key is test_row_0/B:col10/1733177671467/Put/seqid=0 2024-12-02T22:14:33,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742023_1199 (size=12301) 2024-12-02T22:14:33,694 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/7f8276004ad04222a6aebbc4f5d8f96e 2024-12-02T22:14:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/da1244f6f7874e298e002489c3d07d30 is 50, key is test_row_0/C:col10/1733177671467/Put/seqid=0 2024-12-02T22:14:33,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742024_1200 (size=12301) 2024-12-02T22:14:33,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/da1244f6f7874e298e002489c3d07d30 2024-12-02T22:14:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/152481d4f4d6441b9c70e8c222e5177a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a 2024-12-02T22:14:33,724 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a, entries=150, sequenceid=394, filesize=30.5 K 2024-12-02T22:14:33,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/7f8276004ad04222a6aebbc4f5d8f96e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e 2024-12-02T22:14:33,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e, entries=150, sequenceid=394, filesize=12.0 K 2024-12-02T22:14:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/da1244f6f7874e298e002489c3d07d30 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30 2024-12-02T22:14:33,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30, entries=150, sequenceid=394, filesize=12.0 K 2024-12-02T22:14:33,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e76fa5b70a6b892d2b9930c0da4f98cd in 489ms, sequenceid=394, compaction requested=false 2024-12-02T22:14:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T22:14:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T22:14:33,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T22:14:33,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-12-02T22:14:33,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0280 sec 2024-12-02T22:14:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:33,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:33,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ff2082ddff424fa1bdb3fdb0795f20a5_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:33,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742025_1201 (size=14994) 2024-12-02T22:14:33,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177733782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177733785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177733787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177733788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T22:14:33,863 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:33,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T22:14:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:33,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:33,866 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:33,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177733893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177733903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177733903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177733903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:34,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177734103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177734113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177734113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177734113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:34,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:34,194 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ff2082ddff424fa1bdb3fdb0795f20a5_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff2082ddff424fa1bdb3fdb0795f20a5_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:34,195 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3db72637b5aa4f9db722f901e22257d4, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3db72637b5aa4f9db722f901e22257d4 is 175, key is test_row_0/A:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742026_1202 (size=39949) 2024-12-02T22:14:34,201 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3db72637b5aa4f9db722f901e22257d4 2024-12-02T22:14:34,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/046eec200d814d52856fec499585ed7b is 50, key is test_row_0/B:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742027_1203 (size=12301) 2024-12-02T22:14:34,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/046eec200d814d52856fec499585ed7b 2024-12-02T22:14:34,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f9d63e9ae0d04cce99c15a404151debe is 50, key is test_row_0/C:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742028_1204 (size=12301) 2024-12-02T22:14:34,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177734413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177734423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177734423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177734433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:34,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f9d63e9ae0d04cce99c15a404151debe 2024-12-02T22:14:34,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:34,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3db72637b5aa4f9db722f901e22257d4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4 2024-12-02T22:14:34,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4, entries=200, sequenceid=411, filesize=39.0 K 2024-12-02T22:14:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/046eec200d814d52856fec499585ed7b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b 2024-12-02T22:14:34,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b, entries=150, sequenceid=411, filesize=12.0 K 2024-12-02T22:14:34,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/f9d63e9ae0d04cce99c15a404151debe as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe 2024-12-02T22:14:34,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe, entries=150, sequenceid=411, filesize=12.0 K 2024-12-02T22:14:34,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e76fa5b70a6b892d2b9930c0da4f98cd in 886ms, sequenceid=411, compaction requested=true 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:34,652 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:34,652 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:34,653 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:34,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:34,653 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:34,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:34,653 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,653 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,653 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b352231ce9e94936905ec6a14b25d412, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.6 K 2024-12-02T22:14:34,653 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=100.7 K 2024-12-02T22:14:34,653 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4] 2024-12-02T22:14:34,654 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b352231ce9e94936905ec6a14b25d412, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:34,654 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45918b134d2445a9c0d74ca6807613b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:34,654 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8276004ad04222a6aebbc4f5d8f96e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733177671467 2024-12-02T22:14:34,654 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152481d4f4d6441b9c70e8c222e5177a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733177671467 2024-12-02T22:14:34,655 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 046eec200d814d52856fec499585ed7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:34,655 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db72637b5aa4f9db722f901e22257d4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:34,666 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:34,667 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/c3034582cc1a4d53b168da3b16a076cd is 50, key is test_row_0/B:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,677 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,680 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120221d6d0c2e25f4d049988c50a43f12408_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742029_1205 (size=13017) 2024-12-02T22:14:34,682 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120221d6d0c2e25f4d049988c50a43f12408_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,682 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120221d6d0c2e25f4d049988c50a43f12408_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,686 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/c3034582cc1a4d53b168da3b16a076cd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c3034582cc1a4d53b168da3b16a076cd 2024-12-02T22:14:34,692 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into c3034582cc1a4d53b168da3b16a076cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:34,692 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:34,693 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177674652; duration=0sec 2024-12-02T22:14:34,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:34,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:34,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:34,694 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:34,694 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:34,694 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,694 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7650417eb4824bed8b3797d85ba12e6b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.6 K 2024-12-02T22:14:34,694 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7650417eb4824bed8b3797d85ba12e6b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733177671113 2024-12-02T22:14:34,695 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting da1244f6f7874e298e002489c3d07d30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733177671467 2024-12-02T22:14:34,695 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d63e9ae0d04cce99c15a404151debe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:34,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742030_1206 (size=4469) 2024-12-02T22:14:34,699 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#169 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:34,700 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ff071799a5dc416280d9cde8155cef16 is 175, key is test_row_0/A:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742031_1207 (size=31971) 2024-12-02T22:14:34,708 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:34,708 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/0db06a0e0ab946208ba13c04c22c3234 is 50, key is test_row_0/C:col10/1733177673764/Put/seqid=0 2024-12-02T22:14:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742032_1208 (size=13017) 2024-12-02T22:14:34,716 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/0db06a0e0ab946208ba13c04c22c3234 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/0db06a0e0ab946208ba13c04c22c3234 2024-12-02T22:14:34,724 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 0db06a0e0ab946208ba13c04c22c3234(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:34,724 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:34,724 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177674652; duration=0sec 2024-12-02T22:14:34,724 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:34,724 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:34,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T22:14:34,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:34,783 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:34,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021076750a0a7146498b2e09de05b1cf51_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177673784/Put/seqid=0 2024-12-02T22:14:34,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742033_1209 (size=12454) 2024-12-02T22:14:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:34,805 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021076750a0a7146498b2e09de05b1cf51_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021076750a0a7146498b2e09de05b1cf51_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fa1ab6c4c23b43f9a9e9bef917648763, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fa1ab6c4c23b43f9a9e9bef917648763 is 175, key is test_row_0/A:col10/1733177673784/Put/seqid=0 2024-12-02T22:14:34,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742034_1210 (size=31255) 2024-12-02T22:14:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:34,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:34,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177734948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177734953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177734954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177734955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177735063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177735063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177735063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177735064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,110 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/ff071799a5dc416280d9cde8155cef16 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16 2024-12-02T22:14:35,116 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into ff071799a5dc416280d9cde8155cef16(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:35,116 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:35,116 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177674652; duration=0sec 2024-12-02T22:14:35,117 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:35,117 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:35,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fa1ab6c4c23b43f9a9e9bef917648763 2024-12-02T22:14:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9280e5498daf4f909b09e7b179dfa415 is 50, key is test_row_0/B:col10/1733177673784/Put/seqid=0 2024-12-02T22:14:35,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742035_1211 (size=12301) 2024-12-02T22:14:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177735273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177735273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177735273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177735274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177735583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177735583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177735583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177735583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:35,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9280e5498daf4f909b09e7b179dfa415 2024-12-02T22:14:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/3716b64d721a4cd394edd63a2da7634a is 50, key is test_row_0/C:col10/1733177673784/Put/seqid=0 2024-12-02T22:14:35,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742036_1212 (size=12301) 2024-12-02T22:14:35,653 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/3716b64d721a4cd394edd63a2da7634a 2024-12-02T22:14:35,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fa1ab6c4c23b43f9a9e9bef917648763 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763 2024-12-02T22:14:35,666 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763, entries=150, sequenceid=429, filesize=30.5 K 2024-12-02T22:14:35,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/9280e5498daf4f909b09e7b179dfa415 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415 2024-12-02T22:14:35,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415, entries=150, sequenceid=429, filesize=12.0 K 2024-12-02T22:14:35,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/3716b64d721a4cd394edd63a2da7634a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a 2024-12-02T22:14:35,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a, entries=150, sequenceid=429, filesize=12.0 K 2024-12-02T22:14:35,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e76fa5b70a6b892d2b9930c0da4f98cd in 900ms, sequenceid=429, compaction requested=false 2024-12-02T22:14:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T22:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T22:14:35,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T22:14:35,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-12-02T22:14:35,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8240 sec 2024-12-02T22:14:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T22:14:36,013 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:36,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-02T22:14:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:36,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:36,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:36,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:36,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d15bbc84cb17449c8013e458cf2150db_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177736111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177736112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177736113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177736115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742037_1213 (size=12454) 2024-12-02T22:14:36,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:36,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d15bbc84cb17449c8013e458cf2150db_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d15bbc84cb17449c8013e458cf2150db_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:36,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/59ac74ef2ee34190b3369e15ecd12e1f, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/59ac74ef2ee34190b3369e15ecd12e1f is 175, key is test_row_0/A:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742038_1214 (size=31255) 2024-12-02T22:14:36,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/59ac74ef2ee34190b3369e15ecd12e1f 2024-12-02T22:14:36,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/975361a8aa5d4e3ba1219200470617e6 is 50, key is test_row_0/B:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742039_1215 (size=12301) 2024-12-02T22:14:36,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/975361a8aa5d4e3ba1219200470617e6 2024-12-02T22:14:36,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/08efb4669f91480a9488b05dd3eae9c3 is 50, key is test_row_0/C:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742040_1216 (size=12301) 2024-12-02T22:14:36,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T22:14:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:36,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/08efb4669f91480a9488b05dd3eae9c3 2024-12-02T22:14:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/59ac74ef2ee34190b3369e15ecd12e1f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f 2024-12-02T22:14:36,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f, entries=150, sequenceid=449, filesize=30.5 K 2024-12-02T22:14:36,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/975361a8aa5d4e3ba1219200470617e6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6 2024-12-02T22:14:36,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6, entries=150, sequenceid=449, filesize=12.0 K 2024-12-02T22:14:36,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/08efb4669f91480a9488b05dd3eae9c3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3 2024-12-02T22:14:36,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3, entries=150, sequenceid=449, filesize=12.0 K 2024-12-02T22:14:36,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e76fa5b70a6b892d2b9930c0da4f98cd in 102ms, sequenceid=449, compaction requested=true 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:36,198 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:36,198 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:36,207 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:36,207 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:36,207 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:36,207 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:36,207 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,207 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,208 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=92.3 K 2024-12-02T22:14:36,208 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,208 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c3034582cc1a4d53b168da3b16a076cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.7 K 2024-12-02T22:14:36,208 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f] 2024-12-02T22:14:36,208 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3034582cc1a4d53b168da3b16a076cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:36,209 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9280e5498daf4f909b09e7b179dfa415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733177673779 2024-12-02T22:14:36,209 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 975361a8aa5d4e3ba1219200470617e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:36,210 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff071799a5dc416280d9cde8155cef16, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:36,210 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1ab6c4c23b43f9a9e9bef917648763, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733177673779 2024-12-02T22:14:36,211 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ac74ef2ee34190b3369e15ecd12e1f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:36,219 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:36,220 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/8a08ffa127d8499c9c0568dd336f98db is 50, key is test_row_0/B:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,220 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,224 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202f0fb57b12f7b4f908ab3846a376a13cc_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742041_1217 (size=13119) 2024-12-02T22:14:36,227 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202f0fb57b12f7b4f908ab3846a376a13cc_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,227 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f0fb57b12f7b4f908ab3846a376a13cc_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:36,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,234 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/8a08ffa127d8499c9c0568dd336f98db as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/8a08ffa127d8499c9c0568dd336f98db 2024-12-02T22:14:36,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120274246a972f27423582a1d1e2ab088362_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177676113/Put/seqid=0 2024-12-02T22:14:36,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742042_1218 (size=4469) 2024-12-02T22:14:36,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742043_1219 (size=12454) 2024-12-02T22:14:36,243 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into 8a08ffa127d8499c9c0568dd336f98db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:36,243 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:36,243 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=13, startTime=1733177676198; duration=0sec 2024-12-02T22:14:36,244 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:36,244 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:36,244 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:36,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:36,245 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:36,245 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:36,245 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,245 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/0db06a0e0ab946208ba13c04c22c3234, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=36.7 K 2024-12-02T22:14:36,246 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db06a0e0ab946208ba13c04c22c3234, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733177673640 2024-12-02T22:14:36,247 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3716b64d721a4cd394edd63a2da7634a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733177673779 2024-12-02T22:14:36,247 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 08efb4669f91480a9488b05dd3eae9c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:36,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120274246a972f27423582a1d1e2ab088362_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274246a972f27423582a1d1e2ab088362_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:36,250 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e1b14df4a14147889fb81ce8ad42ade8, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:36,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e1b14df4a14147889fb81ce8ad42ade8 is 175, key is test_row_0/A:col10/1733177676113/Put/seqid=0 2024-12-02T22:14:36,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742044_1220 (size=31255) 2024-12-02T22:14:36,259 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:36,260 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/5ab88da7cf764917a1900f0ba16aa83c is 50, key is test_row_0/C:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177736256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=464, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e1b14df4a14147889fb81ce8ad42ade8 2024-12-02T22:14:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177736263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177736265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177736266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/90ec5ca178f4459981268b943f3f2ff7 is 50, key is test_row_0/B:col10/1733177676113/Put/seqid=0 2024-12-02T22:14:36,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742045_1221 (size=13119) 2024-12-02T22:14:36,291 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/5ab88da7cf764917a1900f0ba16aa83c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/5ab88da7cf764917a1900f0ba16aa83c 2024-12-02T22:14:36,297 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into 5ab88da7cf764917a1900f0ba16aa83c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:36,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:36,297 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=13, startTime=1733177676198; duration=0sec 2024-12-02T22:14:36,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:36,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742046_1222 (size=12301) 2024-12-02T22:14:36,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/90ec5ca178f4459981268b943f3f2ff7 2024-12-02T22:14:36,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/cee9919f8eaf4720a1d61e2ca3a5a0df is 50, key is test_row_0/C:col10/1733177676113/Put/seqid=0 2024-12-02T22:14:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742047_1223 (size=12301) 2024-12-02T22:14:36,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T22:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177736373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177736373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177736374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177736383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T22:14:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177736583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177736584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177736584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177736593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T22:14:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:36,640 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:36,641 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/69d2846b13ac4871b1f1da0e45f9dfd3 is 175, key is test_row_0/A:col10/1733177676094/Put/seqid=0 2024-12-02T22:14:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:36,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742048_1224 (size=32073) 2024-12-02T22:14:36,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/cee9919f8eaf4720a1d61e2ca3a5a0df 2024-12-02T22:14:36,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/e1b14df4a14147889fb81ce8ad42ade8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8 2024-12-02T22:14:36,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8, entries=150, sequenceid=464, filesize=30.5 K 2024-12-02T22:14:36,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/90ec5ca178f4459981268b943f3f2ff7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7 2024-12-02T22:14:36,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7, entries=150, sequenceid=464, filesize=12.0 K 2024-12-02T22:14:36,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/cee9919f8eaf4720a1d61e2ca3a5a0df as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df 2024-12-02T22:14:36,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df, entries=150, sequenceid=464, filesize=12.0 K 2024-12-02T22:14:36,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e76fa5b70a6b892d2b9930c0da4f98cd in 515ms, sequenceid=464, compaction requested=false 2024-12-02T22:14:36,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:36,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:36,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:36,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7058388f6a04707bd57884a92bf7d65_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177676261/Put/seqid=0 2024-12-02T22:14:36,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742049_1225 (size=12454) 2024-12-02T22:14:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:36,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. as already flushing 2024-12-02T22:14:36,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177736911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177736913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177736916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177736916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177737024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177737024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177737033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177737033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,065 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/69d2846b13ac4871b1f1da0e45f9dfd3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3 2024-12-02T22:14:37,071 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into 69d2846b13ac4871b1f1da0e45f9dfd3(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:37,071 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:37,071 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=13, startTime=1733177676198; duration=0sec 2024-12-02T22:14:37,071 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:37,071 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:37,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:37,218 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7058388f6a04707bd57884a92bf7d65_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7058388f6a04707bd57884a92bf7d65_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:37,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5d797a1174cb4e72a0b644163101c7dd, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5d797a1174cb4e72a0b644163101c7dd is 175, key is test_row_0/A:col10/1733177676261/Put/seqid=0 2024-12-02T22:14:37,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177737233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742050_1226 (size=31255) 2024-12-02T22:14:37,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5d797a1174cb4e72a0b644163101c7dd 2024-12-02T22:14:37,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177737234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177737243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177737243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b828520405ff4fe9beb9675614dc6328 is 50, key is test_row_0/B:col10/1733177676261/Put/seqid=0 2024-12-02T22:14:37,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742051_1227 (size=12301) 2024-12-02T22:14:37,264 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b828520405ff4fe9beb9675614dc6328 2024-12-02T22:14:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6f48645199624112a8f1bb234cd70398 is 50, key is test_row_0/C:col10/1733177676261/Put/seqid=0 2024-12-02T22:14:37,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742052_1228 (size=12301) 2024-12-02T22:14:37,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47922 deadline: 1733177737543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47950 deadline: 1733177737543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733177737553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47940 deadline: 1733177737553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:14:37,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6f48645199624112a8f1bb234cd70398 2024-12-02T22:14:37,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/5d797a1174cb4e72a0b644163101c7dd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd 2024-12-02T22:14:37,726 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd, entries=150, sequenceid=484, filesize=30.5 K 2024-12-02T22:14:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/b828520405ff4fe9beb9675614dc6328 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328 2024-12-02T22:14:37,735 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328, entries=150, sequenceid=484, filesize=12.0 K 2024-12-02T22:14:37,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6f48645199624112a8f1bb234cd70398 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398 2024-12-02T22:14:37,750 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398, entries=150, sequenceid=484, filesize=12.0 K 2024-12-02T22:14:37,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e76fa5b70a6b892d2b9930c0da4f98cd in 967ms, sequenceid=484, compaction requested=true 2024-12-02T22:14:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-02T22:14:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-02T22:14:37,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T22:14:37,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-12-02T22:14:37,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.7430 sec 2024-12-02T22:14:37,934 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-12-02T22:14:37,934 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-12-02T22:14:37,934 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:37,934 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:37,934 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,934 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,934 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:37,936 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:37,937 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-12-02T22:14:37,937 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:37,937 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,937 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:37,941 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-02T22:14:37,941 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:37,941 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,942 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:37,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:38,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:38,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:38,056 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-02T22:14:38,056 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:38,056 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,056 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:38,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202823139cd99094d2caca5ca47f4fd4880_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:38,065 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-02T22:14:38,065 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-02T22:14:38,065 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:38,065 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:38,065 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,065 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,065 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,065 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:38,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:38,067 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-02T22:14:38,067 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:38,067 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,067 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:38,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742053_1229 (size=12454) 2024-12-02T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T22:14:38,163 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:38,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:38,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202823139cd99094d2caca5ca47f4fd4880_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202823139cd99094d2caca5ca47f4fd4880_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:38,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/251aede489574de093ef3edf3e13f7b5, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:38,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/251aede489574de093ef3edf3e13f7b5 is 175, key is test_row_0/A:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742054_1230 (size=31255) 2024-12-02T22:14:38,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=500, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/251aede489574de093ef3edf3e13f7b5 2024-12-02T22:14:38,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/357047fe9313477f8c3056874f0be743 is 50, key is test_row_0/B:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:38,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742055_1231 (size=12301) 2024-12-02T22:14:38,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/357047fe9313477f8c3056874f0be743 2024-12-02T22:14:38,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6a867dac5a8e4c95b2a8a321fe0a3503 is 50, key is test_row_0/C:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:38,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742056_1232 (size=12301) 2024-12-02T22:14:38,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6a867dac5a8e4c95b2a8a321fe0a3503 2024-12-02T22:14:38,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/251aede489574de093ef3edf3e13f7b5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5 2024-12-02T22:14:38,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5, entries=150, sequenceid=500, filesize=30.5 K 2024-12-02T22:14:38,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/357047fe9313477f8c3056874f0be743 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743 2024-12-02T22:14:38,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743, entries=150, sequenceid=500, filesize=12.0 K 2024-12-02T22:14:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/6a867dac5a8e4c95b2a8a321fe0a3503 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503 2024-12-02T22:14:38,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503, entries=150, sequenceid=500, filesize=12.0 K 2024-12-02T22:14:38,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for e76fa5b70a6b892d2b9930c0da4f98cd in 880ms, sequenceid=500, compaction requested=true 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:38,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e76fa5b70a6b892d2b9930c0da4f98cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:38,935 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:38,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:38,936 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:38,937 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:38,937 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:38,937 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/B is initiating minor compaction (all files) 2024-12-02T22:14:38,937 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/A is initiating minor compaction (all files) 2024-12-02T22:14:38,937 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/B in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:38,937 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/A in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:38,937 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=122.9 K 2024-12-02T22:14:38,937 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/8a08ffa127d8499c9c0568dd336f98db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=48.8 K 2024-12-02T22:14:38,937 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:38,937 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5] 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a08ffa127d8499c9c0568dd336f98db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d2846b13ac4871b1f1da0e45f9dfd3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ec5ca178f4459981268b943f3f2ff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733177676110 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1b14df4a14147889fb81ce8ad42ade8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733177676110 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b828520405ff4fe9beb9675614dc6328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733177676239 2024-12-02T22:14:38,938 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d797a1174cb4e72a0b644163101c7dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733177676239 2024-12-02T22:14:38,939 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 357047fe9313477f8c3056874f0be743, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733177676911 2024-12-02T22:14:38,939 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 251aede489574de093ef3edf3e13f7b5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733177676911 2024-12-02T22:14:38,953 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:38,956 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#B#compaction#189 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:38,957 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e44db6bc008c46efa9f04e51ccc264a4 is 50, key is test_row_0/B:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:38,961 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ed2acf2935904499b8932f141b2ed795_e76fa5b70a6b892d2b9930c0da4f98cd store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:38,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ed2acf2935904499b8932f141b2ed795_e76fa5b70a6b892d2b9930c0da4f98cd, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:38,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ed2acf2935904499b8932f141b2ed795_e76fa5b70a6b892d2b9930c0da4f98cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742057_1233 (size=13255) 2024-12-02T22:14:39,006 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/e44db6bc008c46efa9f04e51ccc264a4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e44db6bc008c46efa9f04e51ccc264a4 2024-12-02T22:14:39,012 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/B of e76fa5b70a6b892d2b9930c0da4f98cd into e44db6bc008c46efa9f04e51ccc264a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:39,012 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:39,012 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/B, priority=12, startTime=1733177678935; duration=0sec 2024-12-02T22:14:39,013 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:39,013 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:B 2024-12-02T22:14:39,013 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:39,014 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:39,014 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): e76fa5b70a6b892d2b9930c0da4f98cd/C is initiating minor compaction (all files) 2024-12-02T22:14:39,014 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e76fa5b70a6b892d2b9930c0da4f98cd/C in TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:39,014 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/5ab88da7cf764917a1900f0ba16aa83c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp, totalSize=48.8 K 2024-12-02T22:14:39,015 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab88da7cf764917a1900f0ba16aa83c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733177674945 2024-12-02T22:14:39,015 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting cee9919f8eaf4720a1d61e2ca3a5a0df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733177676110 2024-12-02T22:14:39,016 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f48645199624112a8f1bb234cd70398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1733177676239 2024-12-02T22:14:39,016 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a867dac5a8e4c95b2a8a321fe0a3503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733177676911 2024-12-02T22:14:39,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742058_1234 (size=4469) 2024-12-02T22:14:39,035 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#A#compaction#190 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:39,036 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fb8db52132a64c7ca5f87a3e81c63139 is 175, key is test_row_0/A:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:39,044 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e76fa5b70a6b892d2b9930c0da4f98cd#C#compaction#191 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:39,045 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/e1470ff0c0fd484bb94d07aab02a5800 is 50, key is test_row_0/C:col10/1733177678054/Put/seqid=0 2024-12-02T22:14:39,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742059_1235 (size=32209) 2024-12-02T22:14:39,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742060_1236 (size=13255) 2024-12-02T22:14:39,491 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/fb8db52132a64c7ca5f87a3e81c63139 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fb8db52132a64c7ca5f87a3e81c63139 2024-12-02T22:14:39,491 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/e1470ff0c0fd484bb94d07aab02a5800 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e1470ff0c0fd484bb94d07aab02a5800 2024-12-02T22:14:39,496 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/A of e76fa5b70a6b892d2b9930c0da4f98cd into fb8db52132a64c7ca5f87a3e81c63139(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:39,496 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e76fa5b70a6b892d2b9930c0da4f98cd/C of e76fa5b70a6b892d2b9930c0da4f98cd into e1470ff0c0fd484bb94d07aab02a5800(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e76fa5b70a6b892d2b9930c0da4f98cd: 2024-12-02T22:14:39,496 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/C, priority=12, startTime=1733177678935; duration=0sec 2024-12-02T22:14:39,496 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., storeName=e76fa5b70a6b892d2b9930c0da4f98cd/A, priority=12, startTime=1733177678935; duration=0sec 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:C 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:39,496 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e76fa5b70a6b892d2b9930c0da4f98cd:A 2024-12-02T22:14:42,485 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-02T22:14:42,485 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:14:42,485 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:42,485 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:42,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2674 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2827 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1206 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3616 rows 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-12-02T22:14:42,486 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3581 rows 2024-12-02T22:14:42,486 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:14:42,486 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:14:42,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:42,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:42,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:14:42,487 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:14:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:42,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177682490"}]},"ts":"1733177682490"} 2024-12-02T22:14:42,492 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:14:42,492 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:14:42,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:14:42,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, UNASSIGN}] 2024-12-02T22:14:42,495 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, UNASSIGN 2024-12-02T22:14:42,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:42,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, UNASSIGN because future has completed 2024-12-02T22:14:42,498 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:14:42,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:42,650 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:42,650 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:14:42,650 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing e76fa5b70a6b892d2b9930c0da4f98cd, disabling compactions & flushes 2024-12-02T22:14:42,650 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:42,650 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:42,650 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. after waiting 0 ms 2024-12-02T22:14:42,650 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:42,650 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing e76fa5b70a6b892d2b9930c0da4f98cd 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=A 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=B 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e76fa5b70a6b892d2b9930c0da4f98cd, store=C 2024-12-02T22:14:42,651 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:42,656 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120286d43e166a674c2192ecc411443d0c1b_e76fa5b70a6b892d2b9930c0da4f98cd is 50, key is test_row_0/A:col10/1733177682484/Put/seqid=0 2024-12-02T22:14:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742061_1237 (size=9914) 2024-12-02T22:14:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:43,061 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:43,067 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120286d43e166a674c2192ecc411443d0c1b_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120286d43e166a674c2192ecc411443d0c1b_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:43,067 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3b5d080c754a4227a37cad9ad1aa3041, store: [table=TestAcidGuarantees family=A region=e76fa5b70a6b892d2b9930c0da4f98cd] 2024-12-02T22:14:43,068 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3b5d080c754a4227a37cad9ad1aa3041 is 175, key is test_row_0/A:col10/1733177682484/Put/seqid=0 2024-12-02T22:14:43,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742062_1238 (size=22561) 2024-12-02T22:14:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:43,478 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=510, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3b5d080c754a4227a37cad9ad1aa3041 2024-12-02T22:14:43,488 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/56b4027cccc84b499304a98e53fe9b96 is 50, key is test_row_0/B:col10/1733177682484/Put/seqid=0 2024-12-02T22:14:43,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742063_1239 (size=9857) 2024-12-02T22:14:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:43,901 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/56b4027cccc84b499304a98e53fe9b96 2024-12-02T22:14:43,908 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/eb0a177fcbb44a328e9f57f05464af9e is 50, key is test_row_0/C:col10/1733177682484/Put/seqid=0 2024-12-02T22:14:43,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742064_1240 (size=9857) 2024-12-02T22:14:44,313 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/eb0a177fcbb44a328e9f57f05464af9e 2024-12-02T22:14:44,318 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/A/3b5d080c754a4227a37cad9ad1aa3041 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3b5d080c754a4227a37cad9ad1aa3041 2024-12-02T22:14:44,322 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3b5d080c754a4227a37cad9ad1aa3041, entries=100, sequenceid=510, filesize=22.0 K 2024-12-02T22:14:44,323 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/B/56b4027cccc84b499304a98e53fe9b96 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/56b4027cccc84b499304a98e53fe9b96 2024-12-02T22:14:44,331 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/56b4027cccc84b499304a98e53fe9b96, entries=100, sequenceid=510, filesize=9.6 K 2024-12-02T22:14:44,332 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/.tmp/C/eb0a177fcbb44a328e9f57f05464af9e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/eb0a177fcbb44a328e9f57f05464af9e 2024-12-02T22:14:44,336 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/eb0a177fcbb44a328e9f57f05464af9e, entries=100, sequenceid=510, filesize=9.6 K 2024-12-02T22:14:44,337 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e76fa5b70a6b892d2b9930c0da4f98cd in 1687ms, sequenceid=510, compaction requested=false 2024-12-02T22:14:44,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5] to archive 2024-12-02T22:14:44,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:44,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/cdf357e0f0474b30beeeb5f6a6db4cdf 2024-12-02T22:14:44,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7fde6e1692be4aa28c6ccf08934ed5db 2024-12-02T22:14:44,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/1bec84565f314defa56f2fbe9f7df7cf 2024-12-02T22:14:44,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e9051e69979b4b96a4cb89c43035bf31 2024-12-02T22:14:44,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/775280c693ed4066a819c13fbddcee00 2024-12-02T22:14:44,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0b0bb450ea114ee4ab3300bc5e758b62 2024-12-02T22:14:44,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/176a543492054680ac3f75d1eb7b36b9 2024-12-02T22:14:44,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/7132cd6abe34487ba6dd4121f9ee6bf2 2024-12-02T22:14:44,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/717a162a24254c388773b76043fcb1b9 2024-12-02T22:14:44,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bb66f2e4b2f4880ab7da8a23e4c63c9 2024-12-02T22:14:44,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/8bc8b85e62a842c1985d26d152d86900 2024-12-02T22:14:44,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/0aa871a0c121422fba860d1e284bf4db 2024-12-02T22:14:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3bbd9fe8d8b84170ad3f6bdea046f8b7 2024-12-02T22:14:44,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/040bc51f1e6440c691b0ef1617264211 2024-12-02T22:14:44,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ed8d0609463843c0b950104b8c608292 2024-12-02T22:14:44,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/06103b7188e842868d9c3a9408d338fa 2024-12-02T22:14:44,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5ae8d84929a04ef596597cac429af513 2024-12-02T22:14:44,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/08817679927840b8be447e1b55520172 2024-12-02T22:14:44,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/c45918b134d2445a9c0d74ca6807613b 2024-12-02T22:14:44,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/152481d4f4d6441b9c70e8c222e5177a 2024-12-02T22:14:44,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3db72637b5aa4f9db722f901e22257d4 2024-12-02T22:14:44,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/ff071799a5dc416280d9cde8155cef16 2024-12-02T22:14:44,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fa1ab6c4c23b43f9a9e9bef917648763 2024-12-02T22:14:44,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/69d2846b13ac4871b1f1da0e45f9dfd3 2024-12-02T22:14:44,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/59ac74ef2ee34190b3369e15ecd12e1f 2024-12-02T22:14:44,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/e1b14df4a14147889fb81ce8ad42ade8 2024-12-02T22:14:44,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/5d797a1174cb4e72a0b644163101c7dd 2024-12-02T22:14:44,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/251aede489574de093ef3edf3e13f7b5 2024-12-02T22:14:44,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c390177347db4aa28aeca477863c81fd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/6fa2ed96799e4dbf92da9f0f063eeaa4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e2d72bb91ff44773bfcd590197c1cc2e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4d7bbc831e3243fa922eb86cfb7c0ef3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/59a38d540fad48949dbd2b81511d43cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b352231ce9e94936905ec6a14b25d412, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c3034582cc1a4d53b168da3b16a076cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/8a08ffa127d8499c9c0568dd336f98db, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743] to archive 2024-12-02T22:14:44,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:44,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b90bceba192941bc94817d00d39ff0da 2024-12-02T22:14:44,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/86eb614d17a84ed6ac2dc00f20c2357a 2024-12-02T22:14:44,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c390177347db4aa28aeca477863c81fd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c390177347db4aa28aeca477863c81fd 2024-12-02T22:14:44,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b8888a2d884e4ec78f46b83cfee72815 2024-12-02T22:14:44,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/ce77e856eec44f5b9c0c7a3d5ca3d97f 2024-12-02T22:14:44,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/6fa2ed96799e4dbf92da9f0f063eeaa4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/6fa2ed96799e4dbf92da9f0f063eeaa4 2024-12-02T22:14:44,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/09c6a34613224bfc94f0b0aed7ca4ef9 2024-12-02T22:14:44,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/65d8b953dc584d8985ec25d013c60067 2024-12-02T22:14:44,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e2d72bb91ff44773bfcd590197c1cc2e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e2d72bb91ff44773bfcd590197c1cc2e 2024-12-02T22:14:44,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9c5eef5fb68c49cd963865e8bb6c7a6f 2024-12-02T22:14:44,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a90fa6675d0b463180d4208b2aa6de6c 2024-12-02T22:14:44,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4d7bbc831e3243fa922eb86cfb7c0ef3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4d7bbc831e3243fa922eb86cfb7c0ef3 2024-12-02T22:14:44,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/4e7922609221495da0c21afef1ff8047 2024-12-02T22:14:44,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/949f524f7b7e442c8d9581c528937af5 2024-12-02T22:14:44,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/59a38d540fad48949dbd2b81511d43cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/59a38d540fad48949dbd2b81511d43cd 2024-12-02T22:14:44,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e0e13fda6bae4bbbad6d404cba79dc61 2024-12-02T22:14:44,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/3da5322808a54a1ba9adec931128040b 2024-12-02T22:14:44,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b352231ce9e94936905ec6a14b25d412 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b352231ce9e94936905ec6a14b25d412 2024-12-02T22:14:44,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/a7c891129abe4657baf8ce22820de5cf 2024-12-02T22:14:44,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/7f8276004ad04222a6aebbc4f5d8f96e 2024-12-02T22:14:44,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c3034582cc1a4d53b168da3b16a076cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/c3034582cc1a4d53b168da3b16a076cd 2024-12-02T22:14:44,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/046eec200d814d52856fec499585ed7b 2024-12-02T22:14:44,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/9280e5498daf4f909b09e7b179dfa415 2024-12-02T22:14:44,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/8a08ffa127d8499c9c0568dd336f98db to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/8a08ffa127d8499c9c0568dd336f98db 2024-12-02T22:14:44,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/975361a8aa5d4e3ba1219200470617e6 2024-12-02T22:14:44,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/90ec5ca178f4459981268b943f3f2ff7 2024-12-02T22:14:44,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/b828520405ff4fe9beb9675614dc6328 2024-12-02T22:14:44,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/357047fe9313477f8c3056874f0be743 2024-12-02T22:14:44,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/663ccbb6b07c4d8e9f6f6f42d2d31faa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/097c369ef8ce4391ab3e9f309600d627, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/de8a24e8064c40a99fef3524bf2f2a6e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e54047f59bf045e5b07bebda07f86b34, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/40b4a777eedd46d4b2d38b3d23930663, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7650417eb4824bed8b3797d85ba12e6b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/0db06a0e0ab946208ba13c04c22c3234, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/5ab88da7cf764917a1900f0ba16aa83c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503] to archive 2024-12-02T22:14:44,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:14:44,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/693afdf779cb4f6480a1f11f8708cc1b 2024-12-02T22:14:44,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/29169608137e489eb32ff2c6efb9d5b1 2024-12-02T22:14:44,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/663ccbb6b07c4d8e9f6f6f42d2d31faa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/663ccbb6b07c4d8e9f6f6f42d2d31faa 2024-12-02T22:14:44,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f5e48befff2a472a8f94608b41b5f774 2024-12-02T22:14:44,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/9fc5062526b245de8522cc674ef9191d 2024-12-02T22:14:44,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/097c369ef8ce4391ab3e9f309600d627 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/097c369ef8ce4391ab3e9f309600d627 2024-12-02T22:14:44,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f485a5439d47439bb9b63430b1c3635f 2024-12-02T22:14:44,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/98e424e395e446c39aa68b1ba88be61d 2024-12-02T22:14:44,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/de8a24e8064c40a99fef3524bf2f2a6e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/de8a24e8064c40a99fef3524bf2f2a6e 2024-12-02T22:14:44,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/bd091215ef014373a330f8280eac8508 2024-12-02T22:14:44,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/46dd99d0c9d64ba385f2e2b2372db52c 2024-12-02T22:14:44,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e54047f59bf045e5b07bebda07f86b34 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e54047f59bf045e5b07bebda07f86b34 2024-12-02T22:14:44,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/55dd91df9cbc4ac48da3dcfabe9a5f18 2024-12-02T22:14:44,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/935726590ed84e10880c10a54a4b4d8b 2024-12-02T22:14:44,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/40b4a777eedd46d4b2d38b3d23930663 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/40b4a777eedd46d4b2d38b3d23930663 2024-12-02T22:14:44,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7aa3b4cc02a24e52b1233bc88bede101 2024-12-02T22:14:44,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/42584cb9459544e69646c3df34611778 2024-12-02T22:14:44,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7650417eb4824bed8b3797d85ba12e6b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/7650417eb4824bed8b3797d85ba12e6b 2024-12-02T22:14:44,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/64407bdf34554a8b87e0cb712f905a6d 2024-12-02T22:14:44,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/da1244f6f7874e298e002489c3d07d30 2024-12-02T22:14:44,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/0db06a0e0ab946208ba13c04c22c3234 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/0db06a0e0ab946208ba13c04c22c3234 2024-12-02T22:14:44,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/f9d63e9ae0d04cce99c15a404151debe 2024-12-02T22:14:44,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/3716b64d721a4cd394edd63a2da7634a 2024-12-02T22:14:44,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/5ab88da7cf764917a1900f0ba16aa83c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/5ab88da7cf764917a1900f0ba16aa83c 2024-12-02T22:14:44,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/08efb4669f91480a9488b05dd3eae9c3 2024-12-02T22:14:44,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/cee9919f8eaf4720a1d61e2ca3a5a0df 2024-12-02T22:14:44,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6f48645199624112a8f1bb234cd70398 2024-12-02T22:14:44,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/6a867dac5a8e4c95b2a8a321fe0a3503 2024-12-02T22:14:44,561 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits/513.seqid, newMaxSeqId=513, maxSeqId=4 2024-12-02T22:14:44,563 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. 2024-12-02T22:14:44,563 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for e76fa5b70a6b892d2b9930c0da4f98cd: Waiting for close lock at 1733177682650Running coprocessor pre-close hooks at 1733177682650Disabling compacts and flushes for region at 1733177682650Disabling writes for close at 1733177682650Obtaining lock to block concurrent updates at 1733177682650Preparing flush snapshotting stores in e76fa5b70a6b892d2b9930c0da4f98cd at 1733177682650Finished memstore snapshotting TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733177682651 (+1 ms)Flushing stores of TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd. at 1733177682651Flushing e76fa5b70a6b892d2b9930c0da4f98cd/A: creating writer at 1733177682651Flushing mob file e76fa5b70a6b892d2b9930c0da4f98cd/A: appending metadata at 1733177682656 (+5 ms)Flushing mob file e76fa5b70a6b892d2b9930c0da4f98cd/A: closing flushed file at 1733177682656Flushing e76fa5b70a6b892d2b9930c0da4f98cd/A: appending metadata at 1733177683068 (+412 ms)Flushing e76fa5b70a6b892d2b9930c0da4f98cd/A: closing flushed file at 1733177683068Flushing e76fa5b70a6b892d2b9930c0da4f98cd/B: creating writer at 1733177683484 (+416 ms)Flushing e76fa5b70a6b892d2b9930c0da4f98cd/B: appending metadata at 1733177683487 (+3 ms)Flushing e76fa5b70a6b892d2b9930c0da4f98cd/B: closing flushed file at 1733177683487Flushing e76fa5b70a6b892d2b9930c0da4f98cd/C: creating writer at 1733177683905 (+418 ms)Flushing e76fa5b70a6b892d2b9930c0da4f98cd/C: appending metadata at 1733177683908 (+3 ms)Flushing e76fa5b70a6b892d2b9930c0da4f98cd/C: closing flushed file at 1733177683908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8e3b8dd: reopening flushed file at 1733177684317 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab075bc: reopening flushed file at 1733177684322 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d03f108: reopening flushed file at 1733177684331 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e76fa5b70a6b892d2b9930c0da4f98cd in 1687ms, sequenceid=510, compaction requested=false at 1733177684337 (+6 ms)Writing region close event to WAL at 1733177684544 (+207 ms)Running coprocessor post-close hooks at 1733177684562 (+18 ms)Closed at 1733177684563 (+1 ms) 2024-12-02T22:14:44,565 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=e76fa5b70a6b892d2b9930c0da4f98cd, regionState=CLOSED 2024-12-02T22:14:44,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:44,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T22:14:44,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure e76fa5b70a6b892d2b9930c0da4f98cd, server=7ae73638d47e,41753,1733177627056 in 2.0710 sec 2024-12-02T22:14:44,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T22:14:44,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e76fa5b70a6b892d2b9930c0da4f98cd, UNASSIGN in 2.0770 sec 2024-12-02T22:14:44,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T22:14:44,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0820 sec 2024-12-02T22:14:44,578 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177684578"}]},"ts":"1733177684578"} 2024-12-02T22:14:44,580 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:14:44,580 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:14:44,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0930 sec 2024-12-02T22:14:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T22:14:44,634 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:44,634 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:14:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,636 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T22:14:44,636 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,640 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,642 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits] 2024-12-02T22:14:44,645 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3b5d080c754a4227a37cad9ad1aa3041 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/3b5d080c754a4227a37cad9ad1aa3041 2024-12-02T22:14:44,646 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fb8db52132a64c7ca5f87a3e81c63139 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/A/fb8db52132a64c7ca5f87a3e81c63139 2024-12-02T22:14:44,648 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/56b4027cccc84b499304a98e53fe9b96 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/56b4027cccc84b499304a98e53fe9b96 2024-12-02T22:14:44,650 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e44db6bc008c46efa9f04e51ccc264a4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/B/e44db6bc008c46efa9f04e51ccc264a4 2024-12-02T22:14:44,651 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e1470ff0c0fd484bb94d07aab02a5800 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/e1470ff0c0fd484bb94d07aab02a5800 2024-12-02T22:14:44,652 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/eb0a177fcbb44a328e9f57f05464af9e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/C/eb0a177fcbb44a328e9f57f05464af9e 2024-12-02T22:14:44,654 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits/513.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd/recovered.edits/513.seqid 2024-12-02T22:14:44,655 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,655 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:14:44,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:14:44,656 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T22:14:44,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120201e1085820474d3d90888b16ccf1a468_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120201e1085820474d3d90888b16ccf1a468_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,661 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202084a46585db6416982f922e513d22d55_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202084a46585db6416982f922e513d22d55_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,662 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020af52c8455174282a1c5a8a601c6b2f9_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020af52c8455174282a1c5a8a601c6b2f9_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021076750a0a7146498b2e09de05b1cf51_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021076750a0a7146498b2e09de05b1cf51_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c9253b02daf4776ac90396f89756020_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021c9253b02daf4776ac90396f89756020_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f0adb1e74304448840d3ed1b00b3964_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f0adb1e74304448840d3ed1b00b3964_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120238eaac49a38e427ab183d780ddf0d514_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120238eaac49a38e427ab183d780ddf0d514_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,668 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202419abc03377943c1af8726472505f37d_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202419abc03377943c1af8726472505f37d_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b963f8939aa43b0a680ef92d3153ab0_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025b963f8939aa43b0a680ef92d3153ab0_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,671 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202617d8c0656854954bec9dbef45150075_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202617d8c0656854954bec9dbef45150075_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,672 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274246a972f27423582a1d1e2ab088362_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120274246a972f27423582a1d1e2ab088362_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,674 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120278a06c0f14de47ddad61761c9d771714_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120278a06c0f14de47ddad61761c9d771714_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,675 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202823139cd99094d2caca5ca47f4fd4880_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202823139cd99094d2caca5ca47f4fd4880_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,677 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120286d43e166a674c2192ecc411443d0c1b_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120286d43e166a674c2192ecc411443d0c1b_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,678 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120289f26509f22b4a3d8fc20adae593ca62_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120289f26509f22b4a3d8fc20adae593ca62_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,679 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c2aee3f3a8384b5abd90c7ddfbf3febf_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c2aee3f3a8384b5abd90c7ddfbf3febf_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,682 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7058388f6a04707bd57884a92bf7d65_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7058388f6a04707bd57884a92bf7d65_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,684 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d15bbc84cb17449c8013e458cf2150db_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d15bbc84cb17449c8013e458cf2150db_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,685 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3e1d0faef1a4b53a2610d6a67c1670a_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3e1d0faef1a4b53a2610d6a67c1670a_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,688 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202faba7a0feb5448d79f56b6fdfa86e654_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202faba7a0feb5448d79f56b6fdfa86e654_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,689 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff2082ddff424fa1bdb3fdb0795f20a5_e76fa5b70a6b892d2b9930c0da4f98cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff2082ddff424fa1bdb3fdb0795f20a5_e76fa5b70a6b892d2b9930c0da4f98cd 2024-12-02T22:14:44,690 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:14:44,692 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,695 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:14:44,697 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:14:44,698 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,698 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:14:44,699 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177684698"}]},"ts":"9223372036854775807"} 2024-12-02T22:14:44,701 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:14:44,701 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e76fa5b70a6b892d2b9930c0da4f98cd, NAME => 'TestAcidGuarantees,,1733177656240.e76fa5b70a6b892d2b9930c0da4f98cd.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:14:44,701 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:14:44,701 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177684701"}]},"ts":"9223372036854775807"} 2024-12-02T22:14:44,702 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:14:44,703 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-12-02T22:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T22:14:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:14:44,744 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:44,755 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1839004683_22 at /127.0.0.1:43018 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=523 (was 523), ProcessCount=11 (was 11), AvailableMemoryMB=8022 (was 7704) - AvailableMemoryMB LEAK? - 2024-12-02T22:14:44,764 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=523, ProcessCount=11, AvailableMemoryMB=8022 2024-12-02T22:14:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:14:44,766 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:14:44,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:14:44,768 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:14:44,768 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:44,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-02T22:14:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T22:14:44,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:14:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742065_1241 (size=1061) 2024-12-02T22:14:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T22:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T22:14:45,178 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:14:45,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742066_1242 (size=53) 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b71ff10c7b688fa2fccab1c4c47e3ef0, disabling compactions & flushes 2024-12-02T22:14:45,185 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. after waiting 0 ms 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,185 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b71ff10c7b688fa2fccab1c4c47e3ef0: Waiting for close lock at 1733177685185Disabling compacts and flushes for region at 1733177685185Disabling writes for close at 1733177685185Writing region close event to WAL at 1733177685185Closed at 1733177685185 2024-12-02T22:14:45,187 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:14:45,188 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177685187"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177685187"}]},"ts":"1733177685187"} 2024-12-02T22:14:45,190 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:14:45,191 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:14:45,191 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177685191"}]},"ts":"1733177685191"} 2024-12-02T22:14:45,192 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:14:45,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, ASSIGN}] 2024-12-02T22:14:45,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, ASSIGN 2024-12-02T22:14:45,195 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:14:45,345 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b71ff10c7b688fa2fccab1c4c47e3ef0, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:45,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, ASSIGN because future has completed 2024-12-02T22:14:45,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:14:45,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:14:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T22:14:45,503 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,503 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:14:45,504 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,504 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:14:45,504 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,504 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,506 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,507 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:45,507 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b71ff10c7b688fa2fccab1c4c47e3ef0 columnFamilyName A 2024-12-02T22:14:45,507 DEBUG [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:45,508 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(327): Store=b71ff10c7b688fa2fccab1c4c47e3ef0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:45,508 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,509 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:45,509 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b71ff10c7b688fa2fccab1c4c47e3ef0 columnFamilyName B 2024-12-02T22:14:45,509 DEBUG [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:45,510 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(327): Store=b71ff10c7b688fa2fccab1c4c47e3ef0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:45,510 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,511 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:14:45,511 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b71ff10c7b688fa2fccab1c4c47e3ef0 columnFamilyName C 2024-12-02T22:14:45,511 DEBUG [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:14:45,512 INFO [StoreOpener-b71ff10c7b688fa2fccab1c4c47e3ef0-1 {}] regionserver.HStore(327): Store=b71ff10c7b688fa2fccab1c4c47e3ef0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:14:45,512 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,512 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,513 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,513 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,514 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,514 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,515 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:14:45,516 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,519 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:14:45,520 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened b71ff10c7b688fa2fccab1c4c47e3ef0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74654780, jitterRate=0.11244291067123413}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:14:45,520 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,522 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for b71ff10c7b688fa2fccab1c4c47e3ef0: Running coprocessor pre-open hook at 1733177685504Writing region info on filesystem at 1733177685504Initializing all the Stores at 1733177685505 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177685505Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177685505Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177685505Cleaning up temporary data from old regions at 1733177685514 (+9 ms)Running coprocessor post-open hooks at 1733177685520 (+6 ms)Region opened successfully at 1733177685522 (+2 ms) 2024-12-02T22:14:45,522 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., pid=63, masterSystemTime=1733177685500 2024-12-02T22:14:45,524 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,524 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:45,525 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b71ff10c7b688fa2fccab1c4c47e3ef0, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:45,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:14:45,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T22:14:45,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 in 180 msec 2024-12-02T22:14:45,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T22:14:45,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, ASSIGN in 337 msec 2024-12-02T22:14:45,532 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:14:45,533 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177685533"}]},"ts":"1733177685533"} 2024-12-02T22:14:45,534 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:14:45,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:14:45,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 769 msec 2024-12-02T22:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T22:14:45,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:45,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207b8823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3fb46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:14:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:14:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8ce003f 2024-12-02T22:14:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:14:45,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:14:45,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e60715b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524a7b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,919 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388b265e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22888c56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c608b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d2d5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b607681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48596e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1fb8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4f377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d64d9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54fd970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f5480e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409f9d9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653e0eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f68f839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8af8ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef4721c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:14:45,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:14:45,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702384f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:14:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:14:45,949 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:14:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5cf981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:14:45,952 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,952 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,953 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:45,953 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,953 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-02T22:14:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:45,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:45,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:45,963 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,972 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,972 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:45,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:45,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,977 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:45,981 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:14:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:14:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:14:45,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177685998, max timestamp=1733177686026Num uniques 859; 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177685998, max timestamp=1733177686026Num uniques 859; 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177685998, max timestamp=1733177686026Num uniques 859; 2024-12-02T22:14:46,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:46,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8054d5a824ba4158b78aa2bc6c96f97a is 50, key is test_row_0/A:col10/1733177685961/Put/seqid=0 2024-12-02T22:14:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:46,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177746065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733177686060, max timestamp=1733177686073Num uniques 1018; 2024-12-02T22:14:46,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-02T22:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733177686060, max timestamp=1733177686073Num uniques 1018; 2024-12-02T22:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-02T22:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733177686060, max timestamp=1733177686073Num uniques 1018; 2024-12-02T22:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-02T22:14:46,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742067_1243 (size=12001) 2024-12-02T22:14:46,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8054d5a824ba4158b78aa2bc6c96f97a 2024-12-02T22:14:46,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177746112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177746117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1b8e606b38a547d584a1183c4e25fb4c is 50, key is test_row_0/B:col10/1733177685961/Put/seqid=0 2024-12-02T22:14:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177746120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177746121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742068_1244 (size=12001) 2024-12-02T22:14:46,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1b8e606b38a547d584a1183c4e25fb4c 2024-12-02T22:14:46,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 is 50, key is test_row_0/C:col10/1733177685961/Put/seqid=0 2024-12-02T22:14:46,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177746183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742069_1245 (size=12001) 2024-12-02T22:14:46,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177746223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177746233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177746233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177746234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177746393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177746433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177746443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177746443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177746443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:46,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 2024-12-02T22:14:46,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8054d5a824ba4158b78aa2bc6c96f97a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a 2024-12-02T22:14:46,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T22:14:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1b8e606b38a547d584a1183c4e25fb4c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c 2024-12-02T22:14:46,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T22:14:46,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 2024-12-02T22:14:46,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T22:14:46,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 655ms, sequenceid=11, compaction requested=false 2024-12-02T22:14:46,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:46,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=140.89 KB heapSize=394.69 KB 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:46,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/a19a2a90db534514b2a04ce4a303ae0e is 50, key is test_row_0/A:col10/1733177686118/Put/seqid=0 2024-12-02T22:14:46,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177746753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177746753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177746754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177746754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177746764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742070_1246 (size=14341) 2024-12-02T22:14:46,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/a19a2a90db534514b2a04ce4a303ae0e 2024-12-02T22:14:46,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/6cb92972607246c6907fd3a9169d5408 is 50, key is test_row_0/B:col10/1733177686118/Put/seqid=0 2024-12-02T22:14:46,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742071_1247 (size=12001) 2024-12-02T22:14:46,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/6cb92972607246c6907fd3a9169d5408 2024-12-02T22:14:46,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/414dfbf833c84af1826bd5f72bcbdc78 is 50, key is test_row_0/C:col10/1733177686118/Put/seqid=0 2024-12-02T22:14:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177746863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:46,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742072_1248 (size=12001) 2024-12-02T22:14:46,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/414dfbf833c84af1826bd5f72bcbdc78 2024-12-02T22:14:46,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/a19a2a90db534514b2a04ce4a303ae0e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e 2024-12-02T22:14:46,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e, entries=200, sequenceid=64, filesize=14.0 K 2024-12-02T22:14:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/6cb92972607246c6907fd3a9169d5408 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408 2024-12-02T22:14:46,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:46,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:46,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:46,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408, entries=150, sequenceid=64, filesize=11.7 K 2024-12-02T22:14:46,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/414dfbf833c84af1826bd5f72bcbdc78 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78 2024-12-02T22:14:46,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78, entries=150, sequenceid=64, filesize=11.7 K 2024-12-02T22:14:46,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~394.64 KB/404112, currentSize=40.25 KB/41220 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 208ms, sequenceid=64, compaction requested=false 2024-12-02T22:14:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:47,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T22:14:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:47,051 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T22:14:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/e6d49d6a2aac4f6e952ad3a96680f56b is 50, key is test_row_0/A:col10/1733177686740/Put/seqid=0 2024-12-02T22:14:47,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:47,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742073_1249 (size=12001) 2024-12-02T22:14:47,125 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/e6d49d6a2aac4f6e952ad3a96680f56b 2024-12-02T22:14:47,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/81fad6c39e7f4986813f78bb6b56833f is 50, key is test_row_0/B:col10/1733177686740/Put/seqid=0 2024-12-02T22:14:47,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742074_1250 (size=12001) 2024-12-02T22:14:47,188 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/81fad6c39e7f4986813f78bb6b56833f 2024-12-02T22:14:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0c1c313e53ec43daa98bc041daaca87d is 50, key is test_row_0/C:col10/1733177686740/Put/seqid=0 2024-12-02T22:14:47,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177687202, max timestamp=1733177687259Num uniques 859; 2024-12-02T22:14:47,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:47,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177687202, max timestamp=1733177687259Num uniques 859; 2024-12-02T22:14:47,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:47,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177687202, max timestamp=1733177687259Num uniques 859; 2024-12-02T22:14:47,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:14:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742075_1251 (size=12001) 2024-12-02T22:14:47,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0c1c313e53ec43daa98bc041daaca87d 2024-12-02T22:14:47,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/e6d49d6a2aac4f6e952ad3a96680f56b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b 2024-12-02T22:14:47,295 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b, entries=150, sequenceid=73, filesize=11.7 K 2024-12-02T22:14:47,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/81fad6c39e7f4986813f78bb6b56833f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f 2024-12-02T22:14:47,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f, entries=150, sequenceid=73, filesize=11.7 K 2024-12-02T22:14:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0c1c313e53ec43daa98bc041daaca87d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d 2024-12-02T22:14:47,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d, entries=150, sequenceid=73, filesize=11.7 K 2024-12-02T22:14:47,312 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=114.05 KB/116790 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 261ms, sequenceid=73, compaction requested=true 2024-12-02T22:14:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:47,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-02T22:14:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-02T22:14:47,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-02T22:14:47,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-12-02T22:14:47,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3660 sec 2024-12-02T22:14:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:47,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:47,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:14:47,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff1ffff968e749fab29964bf0f7b53a8 is 50, key is test_row_0/A:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177747342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177747346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177747348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177747349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177747351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742076_1252 (size=14341) 2024-12-02T22:14:47,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff1ffff968e749fab29964bf0f7b53a8 2024-12-02T22:14:47,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2a2d1412d4f744c7a2ca8ce300ee5731 is 50, key is test_row_0/B:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:47,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742077_1253 (size=12001) 2024-12-02T22:14:47,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2a2d1412d4f744c7a2ca8ce300ee5731 2024-12-02T22:14:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177747463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/830a9f0cb328471a8b19999968f58e5e is 50, key is test_row_0/C:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177747473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177747474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177747474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177747474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742078_1254 (size=12001) 2024-12-02T22:14:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177747673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177747693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177747695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177747695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177747695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:47,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/830a9f0cb328471a8b19999968f58e5e 2024-12-02T22:14:47,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff1ffff968e749fab29964bf0f7b53a8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8 2024-12-02T22:14:47,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8, entries=200, sequenceid=108, filesize=14.0 K 2024-12-02T22:14:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2a2d1412d4f744c7a2ca8ce300ee5731 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731 2024-12-02T22:14:47,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731, entries=150, sequenceid=108, filesize=11.7 K 2024-12-02T22:14:47,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/830a9f0cb328471a8b19999968f58e5e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e 2024-12-02T22:14:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177747983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e, entries=150, sequenceid=108, filesize=11.7 K 2024-12-02T22:14:48,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 682ms, sequenceid=108, compaction requested=true 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:48,003 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:48,003 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:48,009 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:48,009 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:48,009 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,009 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=46.9 K 2024-12-02T22:14:48,009 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:48,009 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:48,009 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,009 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=51.4 K 2024-12-02T22:14:48,009 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8e606b38a547d584a1183c4e25fb4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177685961 2024-12-02T22:14:48,010 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8054d5a824ba4158b78aa2bc6c96f97a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177685961 2024-12-02T22:14:48,010 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb92972607246c6907fd3a9169d5408, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733177686115 2024-12-02T22:14:48,011 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19a2a90db534514b2a04ce4a303ae0e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733177686115 2024-12-02T22:14:48,012 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fad6c39e7f4986813f78bb6b56833f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733177686730 2024-12-02T22:14:48,012 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d49d6a2aac4f6e952ad3a96680f56b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733177686730 2024-12-02T22:14:48,013 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2d1412d4f744c7a2ca8ce300ee5731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:48,014 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1ffff968e749fab29964bf0f7b53a8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:48,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:14:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:48,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/3948884a0a384d3db76d0c9c461942b9 is 50, key is test_row_0/A:col10/1733177687347/Put/seqid=0 2024-12-02T22:14:48,040 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:48,041 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/0692a3263c994ee799b6c9e58178e41d is 50, key is test_row_0/B:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:48,043 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:48,044 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/786d84fd4de24a0d8f22b361840757c6 is 50, key is test_row_0/A:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:48,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742080_1256 (size=12139) 2024-12-02T22:14:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742081_1257 (size=12139) 2024-12-02T22:14:48,087 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/0692a3263c994ee799b6c9e58178e41d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/0692a3263c994ee799b6c9e58178e41d 2024-12-02T22:14:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177748077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177748077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177748077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,094 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 0692a3263c994ee799b6c9e58178e41d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:48,094 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,094 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=12, startTime=1733177688003; duration=0sec 2024-12-02T22:14:48,094 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:48,094 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:48,095 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177748087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,097 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:48,097 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:48,097 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,097 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=46.9 K 2024-12-02T22:14:48,098 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f0d32bd72b43c3b6b7f0a090cf0d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177685961 2024-12-02T22:14:48,098 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 414dfbf833c84af1826bd5f72bcbdc78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733177686115 2024-12-02T22:14:48,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742079_1255 (size=14341) 2024-12-02T22:14:48,101 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1c313e53ec43daa98bc041daaca87d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1733177686730 2024-12-02T22:14:48,101 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 830a9f0cb328471a8b19999968f58e5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T22:14:48,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:48,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:48,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/3948884a0a384d3db76d0c9c461942b9 2024-12-02T22:14:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-02T22:14:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T22:14:48,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:48,108 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:48,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:48,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e958e25eed5c41aaa853971a93e41e36 is 50, key is test_row_0/B:col10/1733177687347/Put/seqid=0 2024-12-02T22:14:48,134 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:14:48,135 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/cf0bb7d2cf2a41daa02ac2b672f1de44 is 50, key is test_row_0/C:col10/1733177687314/Put/seqid=0 2024-12-02T22:14:48,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742082_1258 (size=12001) 2024-12-02T22:14:48,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e958e25eed5c41aaa853971a93e41e36 2024-12-02T22:14:48,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742083_1259 (size=12139) 2024-12-02T22:14:48,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/51f9e39224924e5fb33cf3085a524e16 is 50, key is test_row_0/C:col10/1733177687347/Put/seqid=0 2024-12-02T22:14:48,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177748194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177748194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177748194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742084_1260 (size=12001) 2024-12-02T22:14:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177748204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T22:14:48,229 DEBUG [regionserver/7ae73638d47e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-02T22:14:48,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T22:14:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T22:14:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177748413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177748414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177748414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T22:14:48,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177748423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,482 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/786d84fd4de24a0d8f22b361840757c6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/786d84fd4de24a0d8f22b361840757c6 2024-12-02T22:14:48,488 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into 786d84fd4de24a0d8f22b361840757c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:48,488 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,488 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=12, startTime=1733177688003; duration=0sec 2024-12-02T22:14:48,488 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:48,489 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:48,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177748513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T22:14:48,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:48,578 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/cf0bb7d2cf2a41daa02ac2b672f1de44 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cf0bb7d2cf2a41daa02ac2b672f1de44 2024-12-02T22:14:48,591 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into cf0bb7d2cf2a41daa02ac2b672f1de44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:48,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,591 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=12, startTime=1733177688003; duration=0sec 2024-12-02T22:14:48,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:48,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:48,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/51f9e39224924e5fb33cf3085a524e16 2024-12-02T22:14:48,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/3948884a0a384d3db76d0c9c461942b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9 2024-12-02T22:14:48,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9, entries=200, sequenceid=121, filesize=14.0 K 2024-12-02T22:14:48,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e958e25eed5c41aaa853971a93e41e36 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36 2024-12-02T22:14:48,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T22:14:48,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/51f9e39224924e5fb33cf3085a524e16 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16 2024-12-02T22:14:48,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T22:14:48,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 619ms, sequenceid=121, compaction requested=false 2024-12-02T22:14:48,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T22:14:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,726 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:14:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:48,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:48,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:48,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T22:14:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/44bea61e191145ad903b6f3142029685 is 50, key is test_row_0/A:col10/1733177688083/Put/seqid=0 2024-12-02T22:14:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177748745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177748747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177748750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177748750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742085_1261 (size=12151) 2024-12-02T22:14:48,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/44bea61e191145ad903b6f3142029685 2024-12-02T22:14:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/edadc30363c74516833c9d4dcaa4b883 is 50, key is test_row_0/B:col10/1733177688083/Put/seqid=0 2024-12-02T22:14:48,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742086_1262 (size=12151) 2024-12-02T22:14:48,822 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/edadc30363c74516833c9d4dcaa4b883 2024-12-02T22:14:48,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c3400c03c1d440b5a84bd4968fb4a6fb is 50, key is test_row_0/C:col10/1733177688083/Put/seqid=0 2024-12-02T22:14:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177748864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177748864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177748864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177748864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:48,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742087_1263 (size=12151) 2024-12-02T22:14:48,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c3400c03c1d440b5a84bd4968fb4a6fb 2024-12-02T22:14:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/44bea61e191145ad903b6f3142029685 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685 2024-12-02T22:14:48,908 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T22:14:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/edadc30363c74516833c9d4dcaa4b883 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883 2024-12-02T22:14:48,914 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T22:14:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c3400c03c1d440b5a84bd4968fb4a6fb as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb 2024-12-02T22:14:48,920 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T22:14:48,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 195ms, sequenceid=144, compaction requested=true 2024-12-02T22:14:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-02T22:14:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-02T22:14:48,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T22:14:48,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 820 msec 2024-12-02T22:14:48,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 827 msec 2024-12-02T22:14:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:49,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff3d7f0ecc934502bd649b7fa2602e7a is 50, key is test_row_0/A:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742088_1264 (size=12151) 2024-12-02T22:14:49,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177749123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff3d7f0ecc934502bd649b7fa2602e7a 2024-12-02T22:14:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177749133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177749134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177749136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c8737c17278246c18787ada57372246b is 50, key is test_row_0/B:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742089_1265 (size=12151) 2024-12-02T22:14:49,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c8737c17278246c18787ada57372246b 2024-12-02T22:14:49,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e8328fff9a4e4bc58bbbfa561b24e28e is 50, key is test_row_0/C:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T22:14:49,243 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:49,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T22:14:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T22:14:49,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:49,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:49,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177749243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177749253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177749253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177749254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742090_1266 (size=12151) 2024-12-02T22:14:49,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e8328fff9a4e4bc58bbbfa561b24e28e 2024-12-02T22:14:49,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/ff3d7f0ecc934502bd649b7fa2602e7a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a 2024-12-02T22:14:49,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a, entries=150, sequenceid=159, filesize=11.9 K 2024-12-02T22:14:49,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c8737c17278246c18787ada57372246b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b 2024-12-02T22:14:49,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b, entries=150, sequenceid=159, filesize=11.9 K 2024-12-02T22:14:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/e8328fff9a4e4bc58bbbfa561b24e28e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e 2024-12-02T22:14:49,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e, entries=150, sequenceid=159, filesize=11.9 K 2024-12-02T22:14:49,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 231ms, sequenceid=159, compaction requested=true 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:49,309 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:49,309 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:49,311 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:49,311 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:49,311 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:49,311 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/786d84fd4de24a0d8f22b361840757c6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=49.6 K 2024-12-02T22:14:49,312 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:49,312 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:49,312 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:49,312 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/0692a3263c994ee799b6c9e58178e41d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=47.3 K 2024-12-02T22:14:49,312 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 786d84fd4de24a0d8f22b361840757c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:49,313 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0692a3263c994ee799b6c9e58178e41d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:49,313 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3948884a0a384d3db76d0c9c461942b9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733177687342 2024-12-02T22:14:49,313 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e958e25eed5c41aaa853971a93e41e36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733177687347 2024-12-02T22:14:49,313 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 44bea61e191145ad903b6f3142029685, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733177688075 2024-12-02T22:14:49,314 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting edadc30363c74516833c9d4dcaa4b883, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733177688075 2024-12-02T22:14:49,314 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ff3d7f0ecc934502bd649b7fa2602e7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:49,314 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8737c17278246c18787ada57372246b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:49,336 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:49,337 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1634e7a64e244bae9a7e065248f3e659 is 50, key is test_row_0/A:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,347 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#220 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:49,348 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/da779b57a0874298b6cd09b59ec1731b is 50, key is test_row_0/B:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742091_1267 (size=12425) 2024-12-02T22:14:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T22:14:49,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742092_1268 (size=12425) 2024-12-02T22:14:49,378 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/da779b57a0874298b6cd09b59ec1731b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/da779b57a0874298b6cd09b59ec1731b 2024-12-02T22:14:49,384 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into da779b57a0874298b6cd09b59ec1731b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:49,385 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:49,385 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=12, startTime=1733177689309; duration=0sec 2024-12-02T22:14:49,385 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:49,385 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:49,385 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:49,386 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:49,386 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:49,386 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:49,386 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cf0bb7d2cf2a41daa02ac2b672f1de44, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=47.3 K 2024-12-02T22:14:49,386 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0bb7d2cf2a41daa02ac2b672f1de44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733177687305 2024-12-02T22:14:49,388 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f9e39224924e5fb33cf3085a524e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733177687347 2024-12-02T22:14:49,388 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3400c03c1d440b5a84bd4968fb4a6fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733177688075 2024-12-02T22:14:49,389 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8328fff9a4e4bc58bbbfa561b24e28e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:49,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T22:14:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:49,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/978383e7cf934f70985aec7e7730e942 is 50, key is test_row_0/A:col10/1733177689132/Put/seqid=0 2024-12-02T22:14:49,412 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:49,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/043f5ec4300a409aa01dd0fad10ffb04 is 50, key is test_row_0/C:col10/1733177688747/Put/seqid=0 2024-12-02T22:14:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742094_1270 (size=12151) 2024-12-02T22:14:49,445 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/978383e7cf934f70985aec7e7730e942 2024-12-02T22:14:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742093_1269 (size=12425) 2024-12-02T22:14:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:49,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:49,467 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/043f5ec4300a409aa01dd0fad10ffb04 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/043f5ec4300a409aa01dd0fad10ffb04 2024-12-02T22:14:49,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c2d7f27da9b344089191e8f73adf88ee is 50, key is test_row_0/B:col10/1733177689132/Put/seqid=0 2024-12-02T22:14:49,474 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 043f5ec4300a409aa01dd0fad10ffb04(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:49,474 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:49,474 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=12, startTime=1733177689309; duration=0sec 2024-12-02T22:14:49,474 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:49,474 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177749489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177749491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177749506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177749506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742095_1271 (size=12151) 2024-12-02T22:14:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T22:14:49,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177749563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,569 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in b71ff10c7b688fa2fccab1c4c47e3ef0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:14:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177749613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177749613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177749623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177749623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,773 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1634e7a64e244bae9a7e065248f3e659 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1634e7a64e244bae9a7e065248f3e659 2024-12-02T22:14:49,790 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into 1634e7a64e244bae9a7e065248f3e659(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:49,790 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:49,790 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=12, startTime=1733177689309; duration=0sec 2024-12-02T22:14:49,790 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:49,790 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177749823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177749823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177749833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177749834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T22:14:49,924 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c2d7f27da9b344089191e8f73adf88ee 2024-12-02T22:14:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/be24c105426947f7948c0448994650d2 is 50, key is test_row_0/C:col10/1733177689132/Put/seqid=0 2024-12-02T22:14:49,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742096_1272 (size=12151) 2024-12-02T22:14:49,969 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/be24c105426947f7948c0448994650d2 2024-12-02T22:14:49,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/978383e7cf934f70985aec7e7730e942 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942 2024-12-02T22:14:49,983 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942, entries=150, sequenceid=178, filesize=11.9 K 2024-12-02T22:14:49,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c2d7f27da9b344089191e8f73adf88ee as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee 2024-12-02T22:14:49,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee, entries=150, sequenceid=178, filesize=11.9 K 2024-12-02T22:14:49,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/be24c105426947f7948c0448994650d2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2 2024-12-02T22:14:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2, entries=150, sequenceid=178, filesize=11.9 K 2024-12-02T22:14:50,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 602ms, sequenceid=178, compaction requested=false 2024-12-02T22:14:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T22:14:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T22:14:50,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T22:14:50,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 758 msec 2024-12-02T22:14:50,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 766 msec 2024-12-02T22:14:50,101 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:14:50,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:50,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:50,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:50,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:50,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0f5db566e1d94f42990fd2003dc6421d is 50, key is test_row_0/A:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742097_1273 (size=14541) 2024-12-02T22:14:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177750176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177750184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177750186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177750186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177750293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177750303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177750303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177750303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T22:14:50,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:50,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T22:14:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:50,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:50,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:50,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:50,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177750504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177750513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177750513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177750523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0f5db566e1d94f42990fd2003dc6421d 2024-12-02T22:14:50,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8c5878d2d4a944e8af92a2229458fcb0 is 50, key is test_row_0/B:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742098_1274 (size=12151) 2024-12-02T22:14:50,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8c5878d2d4a944e8af92a2229458fcb0 2024-12-02T22:14:50,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/da67e727ce1d48c2a380dc92f57cd16e is 50, key is test_row_0/C:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742099_1275 (size=12151) 2024-12-02T22:14:50,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/da67e727ce1d48c2a380dc92f57cd16e 2024-12-02T22:14:50,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0f5db566e1d94f42990fd2003dc6421d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d 2024-12-02T22:14:50,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d, entries=200, sequenceid=197, filesize=14.2 K 2024-12-02T22:14:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:50,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8c5878d2d4a944e8af92a2229458fcb0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0 2024-12-02T22:14:50,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0, entries=150, sequenceid=197, filesize=11.9 K 2024-12-02T22:14:50,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/da67e727ce1d48c2a380dc92f57cd16e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e 2024-12-02T22:14:50,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e, entries=150, sequenceid=197, filesize=11.9 K 2024-12-02T22:14:50,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 575ms, sequenceid=197, compaction requested=true 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:50,718 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:50,718 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:50,719 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:50,719 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:50,719 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,719 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1634e7a64e244bae9a7e065248f3e659, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=38.2 K 2024-12-02T22:14:50,720 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:50,720 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:50,720 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,720 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/da779b57a0874298b6cd09b59ec1731b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=35.9 K 2024-12-02T22:14:50,720 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1634e7a64e244bae9a7e065248f3e659, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:50,720 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978383e7cf934f70985aec7e7730e942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733177689080 2024-12-02T22:14:50,721 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting da779b57a0874298b6cd09b59ec1731b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:50,721 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d7f27da9b344089191e8f73adf88ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733177689080 2024-12-02T22:14:50,721 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5db566e1d94f42990fd2003dc6421d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689478 2024-12-02T22:14:50,722 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c5878d2d4a944e8af92a2229458fcb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689487 2024-12-02T22:14:50,741 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:50,741 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3fb02e0cb1dd499ba8cc4d426177a37f is 50, key is test_row_0/B:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,753 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:50,753 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f34733765af3450c97c294c9a7fcb013 is 50, key is test_row_0/A:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742100_1276 (size=12527) 2024-12-02T22:14:50,813 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3fb02e0cb1dd499ba8cc4d426177a37f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3fb02e0cb1dd499ba8cc4d426177a37f 2024-12-02T22:14:50,819 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 3fb02e0cb1dd499ba8cc4d426177a37f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:50,819 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:50,819 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177690718; duration=0sec 2024-12-02T22:14:50,819 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:50,819 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:50,819 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:50,820 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:50,820 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:50,820 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,820 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/043f5ec4300a409aa01dd0fad10ffb04, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=35.9 K 2024-12-02T22:14:50,821 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 043f5ec4300a409aa01dd0fad10ffb04, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733177688747 2024-12-02T22:14:50,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:14:50,821 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting be24c105426947f7948c0448994650d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733177689080 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:50,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:50,822 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting da67e727ce1d48c2a380dc92f57cd16e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689487 2024-12-02T22:14:50,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742101_1277 (size=12527) 2024-12-02T22:14:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:50,836 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f34733765af3450c97c294c9a7fcb013 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f34733765af3450c97c294c9a7fcb013 2024-12-02T22:14:50,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/c9bfecb6b6ba4dc4944275bc4c5cb002 is 50, key is test_row_0/A:col10/1733177690183/Put/seqid=0 2024-12-02T22:14:50,849 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into f34733765af3450c97c294c9a7fcb013(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:50,849 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:50,849 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177690718; duration=0sec 2024-12-02T22:14:50,849 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:50,849 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:50,852 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:50,854 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/9d03c7ba95ec42e798a7796cacdc0f37 is 50, key is test_row_0/C:col10/1733177690141/Put/seqid=0 2024-12-02T22:14:50,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:50,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177750874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177750874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177750890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177750890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:50,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742102_1278 (size=14541) 2024-12-02T22:14:50,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/c9bfecb6b6ba4dc4944275bc4c5cb002 2024-12-02T22:14:50,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742103_1279 (size=12527) 2024-12-02T22:14:50,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/848ace1671ae4385b8f0d2eb133df20c is 50, key is test_row_0/B:col10/1733177690183/Put/seqid=0 2024-12-02T22:14:50,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742104_1280 (size=12151) 2024-12-02T22:14:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177750993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177750994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177751003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177751003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177751213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177751214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177751223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177751223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,340 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/9d03c7ba95ec42e798a7796cacdc0f37 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/9d03c7ba95ec42e798a7796cacdc0f37 2024-12-02T22:14:51,347 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 9d03c7ba95ec42e798a7796cacdc0f37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:51,347 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:51,347 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177690718; duration=0sec 2024-12-02T22:14:51,347 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:51,347 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:51,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/848ace1671ae4385b8f0d2eb133df20c 2024-12-02T22:14:51,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3200cc5205a46579a42d1bef802b83b is 50, key is test_row_0/C:col10/1733177690183/Put/seqid=0 2024-12-02T22:14:51,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742105_1281 (size=12151) 2024-12-02T22:14:51,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3200cc5205a46579a42d1bef802b83b 2024-12-02T22:14:51,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/c9bfecb6b6ba4dc4944275bc4c5cb002 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002 2024-12-02T22:14:51,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002, entries=200, sequenceid=215, filesize=14.2 K 2024-12-02T22:14:51,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/848ace1671ae4385b8f0d2eb133df20c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c 2024-12-02T22:14:51,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c, entries=150, sequenceid=215, filesize=11.9 K 2024-12-02T22:14:51,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3200cc5205a46579a42d1bef802b83b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b 2024-12-02T22:14:51,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b, entries=150, sequenceid=215, filesize=11.9 K 2024-12-02T22:14:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 659ms, sequenceid=215, compaction requested=false 2024-12-02T22:14:51,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:51,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:51,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f28900b7def54c2c94734c0ecdefe732 is 50, key is test_row_0/A:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:51,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742106_1282 (size=12151) 2024-12-02T22:14:51,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f28900b7def54c2c94734c0ecdefe732 2024-12-02T22:14:51,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3d4a86d40539401ca70e1d7cdcb96aa5 is 50, key is test_row_0/B:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:51,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742107_1283 (size=12151) 2024-12-02T22:14:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177751606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177751605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177751608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177751616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177751617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177751725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177751725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177751725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177751733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177751733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177751933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177751934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177751934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:51,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177751943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177751943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3d4a86d40539401ca70e1d7cdcb96aa5 2024-12-02T22:14:52,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/632f2c2070d44b87a8745afcac62f21b is 50, key is test_row_0/C:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:52,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742108_1284 (size=12151) 2024-12-02T22:14:52,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:52,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:52,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177752243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177752243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177752243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177752256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177752256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:52,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/632f2c2070d44b87a8745afcac62f21b 2024-12-02T22:14:52,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f28900b7def54c2c94734c0ecdefe732 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732 2024-12-02T22:14:52,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T22:14:52,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3d4a86d40539401ca70e1d7cdcb96aa5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5 2024-12-02T22:14:52,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T22:14:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/632f2c2070d44b87a8745afcac62f21b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b 2024-12-02T22:14:52,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T22:14:52,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 940ms, sequenceid=233, compaction requested=true 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:52,469 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:52,469 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:52,470 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:52,470 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:52,470 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,470 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:52,470 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:52,470 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f34733765af3450c97c294c9a7fcb013, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=38.3 K 2024-12-02T22:14:52,470 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,470 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3fb02e0cb1dd499ba8cc4d426177a37f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.0 K 2024-12-02T22:14:52,471 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb02e0cb1dd499ba8cc4d426177a37f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689487 2024-12-02T22:14:52,471 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34733765af3450c97c294c9a7fcb013, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689487 2024-12-02T22:14:52,471 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9bfecb6b6ba4dc4944275bc4c5cb002, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733177690173 2024-12-02T22:14:52,471 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 848ace1671ae4385b8f0d2eb133df20c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733177690173 2024-12-02T22:14:52,472 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28900b7def54c2c94734c0ecdefe732, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:52,472 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4a86d40539401ca70e1d7cdcb96aa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:52,492 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:52,493 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/cf7539b102a24885af9d1b75beb3e1d8 is 50, key is test_row_0/B:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:52,507 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:52,507 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/b038fa9ad35a42dabe2f08a99eeb6d3e is 50, key is test_row_0/A:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742109_1285 (size=12629) 2024-12-02T22:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:52,550 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/cf7539b102a24885af9d1b75beb3e1d8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/cf7539b102a24885af9d1b75beb3e1d8 2024-12-02T22:14:52,555 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into cf7539b102a24885af9d1b75beb3e1d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:52,555 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:52,555 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177692469; duration=0sec 2024-12-02T22:14:52,555 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:52,555 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:52,555 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:52,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T22:14:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,556 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:14:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:52,557 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:52,557 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:52,557 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:52,558 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/9d03c7ba95ec42e798a7796cacdc0f37, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.0 K 2024-12-02T22:14:52,559 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d03c7ba95ec42e798a7796cacdc0f37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177689487 2024-12-02T22:14:52,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742110_1286 (size=12629) 2024-12-02T22:14:52,560 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f3200cc5205a46579a42d1bef802b83b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733177690173 2024-12-02T22:14:52,560 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 632f2c2070d44b87a8745afcac62f21b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:52,570 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/b038fa9ad35a42dabe2f08a99eeb6d3e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b038fa9ad35a42dabe2f08a99eeb6d3e 2024-12-02T22:14:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/38cf546f96a1475086ec7b6b1e950bd6 is 50, key is test_row_0/A:col10/1733177691613/Put/seqid=0 2024-12-02T22:14:52,575 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into b038fa9ad35a42dabe2f08a99eeb6d3e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:52,575 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:52,575 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177692469; duration=0sec 2024-12-02T22:14:52,577 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:52,577 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:52,581 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:52,582 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/cc5af2aac19641619ec1240693cf883b is 50, key is test_row_0/C:col10/1733177690868/Put/seqid=0 2024-12-02T22:14:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742111_1287 (size=12151) 2024-12-02T22:14:52,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742112_1288 (size=12629) 2024-12-02T22:14:52,661 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/cc5af2aac19641619ec1240693cf883b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cc5af2aac19641619ec1240693cf883b 2024-12-02T22:14:52,667 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into cc5af2aac19641619ec1240693cf883b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:52,667 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:52,667 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177692469; duration=0sec 2024-12-02T22:14:52,667 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:52,667 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:52,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177752774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177752775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177752778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177752782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177752782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177752893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177752893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177752893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177752893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177752897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,031 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/38cf546f96a1475086ec7b6b1e950bd6 2024-12-02T22:14:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/22126585d83646ecb29f47bb47e808da is 50, key is test_row_0/B:col10/1733177691613/Put/seqid=0 2024-12-02T22:14:53,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742113_1289 (size=12151) 2024-12-02T22:14:53,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177753103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177753104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177753104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177753104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177753104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177753413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177753413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177753413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177753413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177753414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,472 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/22126585d83646ecb29f47bb47e808da 2024-12-02T22:14:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7579a45db77b4cde9a5ecf077143a766 is 50, key is test_row_0/C:col10/1733177691613/Put/seqid=0 2024-12-02T22:14:53,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742114_1290 (size=12151) 2024-12-02T22:14:53,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7579a45db77b4cde9a5ecf077143a766 2024-12-02T22:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/38cf546f96a1475086ec7b6b1e950bd6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6 2024-12-02T22:14:53,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177753923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177753924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177753924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6, entries=150, sequenceid=251, filesize=11.9 K 2024-12-02T22:14:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177753924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177753925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/22126585d83646ecb29f47bb47e808da as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da 2024-12-02T22:14:53,938 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da, entries=150, sequenceid=251, filesize=11.9 K 2024-12-02T22:14:53,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T22:14:53,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7579a45db77b4cde9a5ecf077143a766 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766 2024-12-02T22:14:53,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766, entries=150, sequenceid=251, filesize=11.9 K 2024-12-02T22:14:53,962 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 1406ms, sequenceid=251, compaction requested=false 2024-12-02T22:14:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T22:14:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T22:14:53,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T22:14:53,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5750 sec 2024-12-02T22:14:53,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 3.5840 sec 2024-12-02T22:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T22:14:54,563 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:54,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T22:14:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:54,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:54,567 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:54,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:54,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:54,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/7b7365fdbd254960ad57502b546a7c13 is 50, key is test_row_0/A:col10/1733177692778/Put/seqid=0 2024-12-02T22:14:54,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742115_1291 (size=12301) 2024-12-02T22:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177754956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177754958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:54,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177754959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177754961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177754962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:54,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177755073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177755074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177755074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177755074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177755074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,145 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/7b7365fdbd254960ad57502b546a7c13 2024-12-02T22:14:55,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/f02b7be47d544251a2911aab368b2aad is 50, key is test_row_0/B:col10/1733177692778/Put/seqid=0 2024-12-02T22:14:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742116_1292 (size=12301) 2024-12-02T22:14:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177755284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177755284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177755284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177755285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177755285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/f02b7be47d544251a2911aab368b2aad 2024-12-02T22:14:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177755594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177755594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177755594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177755594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177755595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/a38e23e493284d54b6f67242ffd1eb76 is 50, key is test_row_0/C:col10/1733177692778/Put/seqid=0 2024-12-02T22:14:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742117_1293 (size=12301) 2024-12-02T22:14:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:56,067 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/a38e23e493284d54b6f67242ffd1eb76 2024-12-02T22:14:56,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/7b7365fdbd254960ad57502b546a7c13 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13 2024-12-02T22:14:56,080 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13, entries=150, sequenceid=269, filesize=12.0 K 2024-12-02T22:14:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/f02b7be47d544251a2911aab368b2aad as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad 2024-12-02T22:14:56,087 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad, entries=150, sequenceid=269, filesize=12.0 K 2024-12-02T22:14:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/a38e23e493284d54b6f67242ffd1eb76 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76 2024-12-02T22:14:56,092 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76, entries=150, sequenceid=269, filesize=12.0 K 2024-12-02T22:14:56,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 1372ms, sequenceid=269, compaction requested=true 2024-12-02T22:14:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T22:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T22:14:56,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T22:14:56,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-12-02T22:14:56,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5340 sec 2024-12-02T22:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:56,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:56,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/148c78af3cb64b1fa36d46ff5bb98593 is 50, key is test_row_0/A:col10/1733177696103/Put/seqid=0 2024-12-02T22:14:56,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742118_1294 (size=12301) 2024-12-02T22:14:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177756125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177756129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177756130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177756130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177756132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177756453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177756453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177756454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177756454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177756454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/148c78af3cb64b1fa36d46ff5bb98593 2024-12-02T22:14:56,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/795a29e5ef524d15a1522a4550b202e2 is 50, key is test_row_0/B:col10/1733177696103/Put/seqid=0 2024-12-02T22:14:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742119_1295 (size=12301) 2024-12-02T22:14:56,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/795a29e5ef524d15a1522a4550b202e2 2024-12-02T22:14:56,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f86cb1970c0f48e6a41293137d44e391 is 50, key is test_row_0/C:col10/1733177696103/Put/seqid=0 2024-12-02T22:14:56,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742120_1296 (size=12301) 2024-12-02T22:14:56,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f86cb1970c0f48e6a41293137d44e391 2024-12-02T22:14:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/148c78af3cb64b1fa36d46ff5bb98593 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593 2024-12-02T22:14:56,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T22:14:56,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/795a29e5ef524d15a1522a4550b202e2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2 2024-12-02T22:14:56,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T22:14:56,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f86cb1970c0f48e6a41293137d44e391 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391 2024-12-02T22:14:56,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391, entries=150, sequenceid=286, filesize=12.0 K 2024-12-02T22:14:56,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 540ms, sequenceid=286, compaction requested=true 2024-12-02T22:14:56,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:56,645 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:56,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:56,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:56,646 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:56,647 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:56,648 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:56,648 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:56,648 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b038fa9ad35a42dabe2f08a99eeb6d3e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=48.2 K 2024-12-02T22:14:56,648 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:56,648 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:56,648 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:56,648 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/cf7539b102a24885af9d1b75beb3e1d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=48.2 K 2024-12-02T22:14:56,649 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b038fa9ad35a42dabe2f08a99eeb6d3e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:56,649 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7539b102a24885af9d1b75beb3e1d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:56,649 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 38cf546f96a1475086ec7b6b1e950bd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733177691604 2024-12-02T22:14:56,649 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22126585d83646ecb29f47bb47e808da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733177691604 2024-12-02T22:14:56,650 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7365fdbd254960ad57502b546a7c13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733177692767 2024-12-02T22:14:56,650 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02b7be47d544251a2911aab368b2aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733177692767 2024-12-02T22:14:56,650 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 148c78af3cb64b1fa36d46ff5bb98593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177694951 2024-12-02T22:14:56,650 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 795a29e5ef524d15a1522a4550b202e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177694951 2024-12-02T22:14:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:56,679 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:56,680 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/402d1844d88744a4be5efe1e90a4b831 is 50, key is test_row_0/B:col10/1733177696103/Put/seqid=0 2024-12-02T22:14:56,699 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:56,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/cf98a4bbc7604886b844ae4393243113 is 50, key is test_row_0/A:col10/1733177696103/Put/seqid=0 2024-12-02T22:14:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T22:14:56,713 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:56,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T22:14:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T22:14:56,716 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:56,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:56,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:56,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742121_1297 (size=12915) 2024-12-02T22:14:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:56,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:56,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1b5eab5ed17a429fa991c9586362fe51 is 50, key is test_row_0/A:col10/1733177696778/Put/seqid=0 2024-12-02T22:14:56,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742122_1298 (size=12915) 2024-12-02T22:14:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177756805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T22:14:56,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177756814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177756814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177756816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177756816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742123_1299 (size=17181) 2024-12-02T22:14:56,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1b5eab5ed17a429fa991c9586362fe51 2024-12-02T22:14:56,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/98daff4d6da849a997903e061aacf346 is 50, key is test_row_0/B:col10/1733177696778/Put/seqid=0 2024-12-02T22:14:56,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T22:14:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:56,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:56,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742124_1300 (size=12301) 2024-12-02T22:14:56,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/98daff4d6da849a997903e061aacf346 2024-12-02T22:14:56,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/186f22f68e67495d9a03ad512b906742 is 50, key is test_row_0/C:col10/1733177696778/Put/seqid=0 2024-12-02T22:14:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177756923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177756933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177756934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177756934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177756934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:56,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742125_1301 (size=12301) 2024-12-02T22:14:56,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/186f22f68e67495d9a03ad512b906742 2024-12-02T22:14:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1b5eab5ed17a429fa991c9586362fe51 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51 2024-12-02T22:14:56,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51, entries=250, sequenceid=303, filesize=16.8 K 2024-12-02T22:14:56,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/98daff4d6da849a997903e061aacf346 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346 2024-12-02T22:14:56,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346, entries=150, sequenceid=303, filesize=12.0 K 2024-12-02T22:14:56,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/186f22f68e67495d9a03ad512b906742 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742 2024-12-02T22:14:56,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742, entries=150, sequenceid=303, filesize=12.0 K 2024-12-02T22:14:56,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 185ms, sequenceid=303, compaction requested=true 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T22:14:57,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T22:14:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T22:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/821d767889014038b4de22b5d10f14bc is 50, key is test_row_0/A:col10/1733177696813/Put/seqid=0 2024-12-02T22:14:57,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742126_1302 (size=12301) 2024-12-02T22:14:57,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/821d767889014038b4de22b5d10f14bc 2024-12-02T22:14:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e4c3038788fa439a86e654caf431a9e7 is 50, key is test_row_0/B:col10/1733177696813/Put/seqid=0 2024-12-02T22:14:57,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742127_1303 (size=12301) 2024-12-02T22:14:57,137 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/402d1844d88744a4be5efe1e90a4b831 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/402d1844d88744a4be5efe1e90a4b831 2024-12-02T22:14:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:57,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:57,142 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 402d1844d88744a4be5efe1e90a4b831(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:57,142 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:57,142 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=12, startTime=1733177696646; duration=0sec 2024-12-02T22:14:57,142 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T22:14:57,143 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:57,143 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-02T22:14:57,145 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:14:57,145 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:14:57,145 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:14:57,145 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:57,145 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:14:57,147 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61683 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:14:57,147 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:57,147 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:57,147 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cc5af2aac19641619ec1240693cf883b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=60.2 K 2024-12-02T22:14:57,148 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5af2aac19641619ec1240693cf883b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733177690868 2024-12-02T22:14:57,148 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7579a45db77b4cde9a5ecf077143a766, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733177691604 2024-12-02T22:14:57,149 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38e23e493284d54b6f67242ffd1eb76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733177692767 2024-12-02T22:14:57,149 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86cb1970c0f48e6a41293137d44e391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177694951 2024-12-02T22:14:57,149 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 186f22f68e67495d9a03ad512b906742, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733177696130 2024-12-02T22:14:57,162 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:57,163 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/98760e799bda4226a4c212ca8883582c is 50, key is test_row_0/C:col10/1733177696778/Put/seqid=0 2024-12-02T22:14:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177757165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742128_1304 (size=12949) 2024-12-02T22:14:57,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177757172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177757174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177757176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177757176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,206 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/cf98a4bbc7604886b844ae4393243113 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/cf98a4bbc7604886b844ae4393243113 2024-12-02T22:14:57,211 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into cf98a4bbc7604886b844ae4393243113(size=12.6 K), total size for store is 29.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:57,211 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=12, startTime=1733177696645; duration=0sec 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:57,211 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:14:57,212 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:14:57,212 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:14:57,212 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:14:57,212 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:57,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177757283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177757294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177757294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177757294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177757294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T22:14:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177757493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177757503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177757503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177757503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177757504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e4c3038788fa439a86e654caf431a9e7 2024-12-02T22:14:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0583e1f053704d4eb94c5ba76f31890c is 50, key is test_row_0/C:col10/1733177696813/Put/seqid=0 2024-12-02T22:14:57,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742129_1305 (size=12301) 2024-12-02T22:14:57,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0583e1f053704d4eb94c5ba76f31890c 2024-12-02T22:14:57,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/821d767889014038b4de22b5d10f14bc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc 2024-12-02T22:14:57,577 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T22:14:57,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/e4c3038788fa439a86e654caf431a9e7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7 2024-12-02T22:14:57,588 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/98760e799bda4226a4c212ca8883582c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/98760e799bda4226a4c212ca8883582c 2024-12-02T22:14:57,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T22:14:57,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/0583e1f053704d4eb94c5ba76f31890c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c 2024-12-02T22:14:57,595 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 98760e799bda4226a4c212ca8883582c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:57,596 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:57,596 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=11, startTime=1733177696966; duration=0sec 2024-12-02T22:14:57,596 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:57,596 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:57,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T22:14:57,597 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 571ms, sequenceid=319, compaction requested=true 2024-12-02T22:14:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T22:14:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T22:14:57,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T22:14:57,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 881 msec 2024-12-02T22:14:57,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 887 msec 2024-12-02T22:14:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:14:57,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:57,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:57,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/5c0b80dc35b3413f9b92a5b56df66b30 is 50, key is test_row_0/A:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:57,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177757832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177757834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177757838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177757840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177757840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T22:14:57,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:14:57,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T22:14:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:14:57,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:14:57,857 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:14:57,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:14:57,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742130_1306 (size=14741) 2024-12-02T22:14:57,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177757943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177757943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177757953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177757954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177757954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:14:58,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177758153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177758153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177758163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177758164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177758164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:14:58,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/5c0b80dc35b3413f9b92a5b56df66b30 2024-12-02T22:14:58,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2c4073a008cc4f1c9619834a13249b00 is 50, key is test_row_0/B:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:58,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742131_1307 (size=12301) 2024-12-02T22:14:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177758463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177758463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177758474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177758474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177758474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:14:58,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2c4073a008cc4f1c9619834a13249b00 2024-12-02T22:14:58,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f270a83035954a4ba1bcc08f60e1c7f8 is 50, key is test_row_0/C:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:58,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742132_1308 (size=12301) 2024-12-02T22:14:58,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:58,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177758973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177758973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177758983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177758983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:14:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:14:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177758993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:14:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:14:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:14:59,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:14:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:14:59,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f270a83035954a4ba1bcc08f60e1c7f8 2024-12-02T22:14:59,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/5c0b80dc35b3413f9b92a5b56df66b30 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30 2024-12-02T22:14:59,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30, entries=200, sequenceid=340, filesize=14.4 K 2024-12-02T22:14:59,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/2c4073a008cc4f1c9619834a13249b00 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00 2024-12-02T22:14:59,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T22:14:59,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f270a83035954a4ba1bcc08f60e1c7f8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8 2024-12-02T22:14:59,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T22:14:59,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 1413ms, sequenceid=340, compaction requested=true 2024-12-02T22:14:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:14:59,220 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:59,220 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:14:59,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:14:59,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:59,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:14:59,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:59,222 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:59,222 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:14:59,222 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,222 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/cf98a4bbc7604886b844ae4393243113, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=55.8 K 2024-12-02T22:14:59,222 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:14:59,222 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:14:59,222 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,222 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/402d1844d88744a4be5efe1e90a4b831, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=48.7 K 2024-12-02T22:14:59,223 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf98a4bbc7604886b844ae4393243113, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177694951 2024-12-02T22:14:59,223 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 402d1844d88744a4be5efe1e90a4b831, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733177694951 2024-12-02T22:14:59,223 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b5eab5ed17a429fa991c9586362fe51, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733177696111 2024-12-02T22:14:59,223 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 98daff4d6da849a997903e061aacf346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733177696130 2024-12-02T22:14:59,223 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821d767889014038b4de22b5d10f14bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733177696798 2024-12-02T22:14:59,224 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0b80dc35b3413f9b92a5b56df66b30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:14:59,224 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c3038788fa439a86e654caf431a9e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733177696798 2024-12-02T22:14:59,224 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4073a008cc4f1c9619834a13249b00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:14:59,238 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:59,239 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#262 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:59,239 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/654e669f82754587a9cd127d3e1fc98a is 50, key is test_row_0/A:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:59,239 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/dc679ace3e6544cfb11a4645862c76dd is 50, key is test_row_0/B:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:59,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742134_1310 (size=13051) 2024-12-02T22:14:59,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742133_1309 (size=13051) 2024-12-02T22:14:59,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T22:14:59,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:14:59,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:14:59,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:14:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:14:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:14:59,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/428776b26175426ba43922f420b1b42d is 50, key is test_row_0/A:col10/1733177697832/Put/seqid=0 2024-12-02T22:14:59,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/654e669f82754587a9cd127d3e1fc98a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/654e669f82754587a9cd127d3e1fc98a 2024-12-02T22:14:59,264 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/dc679ace3e6544cfb11a4645862c76dd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/dc679ace3e6544cfb11a4645862c76dd 2024-12-02T22:14:59,267 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into 654e669f82754587a9cd127d3e1fc98a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:59,267 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:59,267 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=12, startTime=1733177699220; duration=0sec 2024-12-02T22:14:59,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:14:59,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:14:59,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:14:59,269 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:14:59,270 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:14:59,270 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,270 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/98760e799bda4226a4c212ca8883582c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.7 K 2024-12-02T22:14:59,271 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98760e799bda4226a4c212ca8883582c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733177696130 2024-12-02T22:14:59,272 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into dc679ace3e6544cfb11a4645862c76dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:59,272 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0583e1f053704d4eb94c5ba76f31890c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733177696798 2024-12-02T22:14:59,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:59,272 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=12, startTime=1733177699220; duration=0sec 2024-12-02T22:14:59,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:59,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:14:59,272 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f270a83035954a4ba1bcc08f60e1c7f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:14:59,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742135_1311 (size=12301) 2024-12-02T22:14:59,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/428776b26175426ba43922f420b1b42d 2024-12-02T22:14:59,294 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:14:59,295 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/d26c51da3ba4464cbd61b3d49b51e58a is 50, key is test_row_0/C:col10/1733177697158/Put/seqid=0 2024-12-02T22:14:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/df213d3c0a6f4166a91d1be5fb9e3e4e is 50, key is test_row_0/B:col10/1733177697832/Put/seqid=0 2024-12-02T22:14:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742136_1312 (size=13051) 2024-12-02T22:14:59,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742137_1313 (size=12301) 2024-12-02T22:14:59,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/df213d3c0a6f4166a91d1be5fb9e3e4e 2024-12-02T22:14:59,347 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/d26c51da3ba4464cbd61b3d49b51e58a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d26c51da3ba4464cbd61b3d49b51e58a 2024-12-02T22:14:59,356 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into d26c51da3ba4464cbd61b3d49b51e58a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:14:59,356 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:59,356 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177699221; duration=0sec 2024-12-02T22:14:59,356 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:14:59,356 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:14:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c5f8e9f85cb2489ea4241c25e0efea4d is 50, key is test_row_0/C:col10/1733177697832/Put/seqid=0 2024-12-02T22:14:59,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742138_1314 (size=12301) 2024-12-02T22:14:59,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c5f8e9f85cb2489ea4241c25e0efea4d 2024-12-02T22:14:59,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/428776b26175426ba43922f420b1b42d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d 2024-12-02T22:14:59,394 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T22:14:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/df213d3c0a6f4166a91d1be5fb9e3e4e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e 2024-12-02T22:14:59,400 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T22:14:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c5f8e9f85cb2489ea4241c25e0efea4d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d 2024-12-02T22:14:59,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T22:14:59,408 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 158ms, sequenceid=354, compaction requested=false 2024-12-02T22:14:59,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:14:59,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:14:59,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T22:14:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T22:14:59,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T22:14:59,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-12-02T22:14:59,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5660 sec 2024-12-02T22:15:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:00,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0c39e30f6faf4cbf85329d6fcf709aa7 is 50, key is test_row_0/A:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T22:15:00,013 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:00,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T22:15:00,016 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T22:15:00,017 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:00,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:00,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742139_1315 (size=9857) 2024-12-02T22:15:00,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0c39e30f6faf4cbf85329d6fcf709aa7 2024-12-02T22:15:00,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/9449d51651bd44bb858b29a081781b96 is 50, key is test_row_0/B:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177700031, max timestamp=1733177700056Num uniques 859; 2024-12-02T22:15:00,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:00,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177700031, max timestamp=1733177700056Num uniques 859; 2024-12-02T22:15:00,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177700045, max timestamp=1733177700058Num uniques 859; 2024-12-02T22:15:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:00,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742140_1316 (size=9857) 2024-12-02T22:15:00,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/9449d51651bd44bb858b29a081781b96 2024-12-02T22:15:00,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/db02a8b75d424491a877d9037b26a03a is 50, key is test_row_0/C:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742141_1317 (size=9857) 2024-12-02T22:15:00,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/db02a8b75d424491a877d9037b26a03a 2024-12-02T22:15:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/0c39e30f6faf4cbf85329d6fcf709aa7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7 2024-12-02T22:15:00,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7, entries=100, sequenceid=367, filesize=9.6 K 2024-12-02T22:15:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177760096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/9449d51651bd44bb858b29a081781b96 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96 2024-12-02T22:15:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177760100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177760095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177760103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96, entries=100, sequenceid=367, filesize=9.6 K 2024-12-02T22:15:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/db02a8b75d424491a877d9037b26a03a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a 2024-12-02T22:15:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177760111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T22:15:00,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a, entries=100, sequenceid=367, filesize=9.6 K 2024-12-02T22:15:00,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 132ms, sequenceid=367, compaction requested=true 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:00,138 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:00,138 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:00,139 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:00,140 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:15:00,140 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:00,140 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/654e669f82754587a9cd127d3e1fc98a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=34.4 K 2024-12-02T22:15:00,140 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:00,140 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:15:00,140 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:00,140 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654e669f82754587a9cd127d3e1fc98a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:15:00,140 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/dc679ace3e6544cfb11a4645862c76dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=34.4 K 2024-12-02T22:15:00,141 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting dc679ace3e6544cfb11a4645862c76dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:15:00,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428776b26175426ba43922f420b1b42d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733177697832 2024-12-02T22:15:00,141 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting df213d3c0a6f4166a91d1be5fb9e3e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733177697832 2024-12-02T22:15:00,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c39e30f6faf4cbf85329d6fcf709aa7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177700001 2024-12-02T22:15:00,142 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9449d51651bd44bb858b29a081781b96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177700001 2024-12-02T22:15:00,165 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:00,166 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8f4729ef89d44c2dbc12f7cf0fb86621 is 50, key is test_row_0/B:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T22:15:00,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:00,171 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:00,175 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:00,175 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/a746a10eefaa4415be8beaab14694873 is 50, key is test_row_0/A:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/281513b04ff04773b328edf4112e5abd is 50, key is test_row_0/A:col10/1733177700108/Put/seqid=0 2024-12-02T22:15:00,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742142_1318 (size=13153) 2024-12-02T22:15:00,226 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8f4729ef89d44c2dbc12f7cf0fb86621 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8f4729ef89d44c2dbc12f7cf0fb86621 2024-12-02T22:15:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:00,237 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 8f4729ef89d44c2dbc12f7cf0fb86621(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:00,238 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:00,238 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177700138; duration=0sec 2024-12-02T22:15:00,238 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:00,238 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:00,238 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:00,239 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:00,239 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:15:00,239 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:00,239 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d26c51da3ba4464cbd61b3d49b51e58a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=34.4 K 2024-12-02T22:15:00,239 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d26c51da3ba4464cbd61b3d49b51e58a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733177697158 2024-12-02T22:15:00,240 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f8e9f85cb2489ea4241c25e0efea4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733177697832 2024-12-02T22:15:00,240 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting db02a8b75d424491a877d9037b26a03a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177700001 2024-12-02T22:15:00,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742143_1319 (size=13153) 2024-12-02T22:15:00,252 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/a746a10eefaa4415be8beaab14694873 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a746a10eefaa4415be8beaab14694873 2024-12-02T22:15:00,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177760236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177760237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177760244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,258 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into a746a10eefaa4415be8beaab14694873(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:00,258 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:00,258 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177700138; duration=0sec 2024-12-02T22:15:00,258 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:00,258 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:00,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742144_1320 (size=12301) 2024-12-02T22:15:00,262 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:00,262 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c6331283c59b422285f0b865b78b3224 is 50, key is test_row_0/C:col10/1733177700001/Put/seqid=0 2024-12-02T22:15:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177760247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177760253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742145_1321 (size=13153) 2024-12-02T22:15:00,291 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/c6331283c59b422285f0b865b78b3224 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c6331283c59b422285f0b865b78b3224 2024-12-02T22:15:00,296 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into c6331283c59b422285f0b865b78b3224(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:00,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:00,297 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177700138; duration=0sec 2024-12-02T22:15:00,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:00,297 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T22:15:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177760364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177760364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177760364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177760373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177760373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177760574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177760574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177760576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177760583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177760583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T22:15:00,662 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/281513b04ff04773b328edf4112e5abd 2024-12-02T22:15:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1e12ca5ee15f454a90977d2040586607 is 50, key is test_row_0/B:col10/1733177700108/Put/seqid=0 2024-12-02T22:15:00,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742146_1322 (size=12301) 2024-12-02T22:15:00,753 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1e12ca5ee15f454a90977d2040586607 2024-12-02T22:15:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/50bbce0b049b4c14887d94feeb5e0515 is 50, key is test_row_0/C:col10/1733177700108/Put/seqid=0 2024-12-02T22:15:00,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742147_1323 (size=12301) 2024-12-02T22:15:00,826 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/50bbce0b049b4c14887d94feeb5e0515 2024-12-02T22:15:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/281513b04ff04773b328edf4112e5abd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd 2024-12-02T22:15:00,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd, entries=150, sequenceid=403, filesize=12.0 K 2024-12-02T22:15:00,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1e12ca5ee15f454a90977d2040586607 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607 2024-12-02T22:15:00,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607, entries=150, sequenceid=403, filesize=12.0 K 2024-12-02T22:15:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/50bbce0b049b4c14887d94feeb5e0515 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515 2024-12-02T22:15:00,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515, entries=150, sequenceid=403, filesize=12.0 K 2024-12-02T22:15:00,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 684ms, sequenceid=403, compaction requested=false 2024-12-02T22:15:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T22:15:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T22:15:00,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T22:15:00,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-12-02T22:15:00,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 846 msec 2024-12-02T22:15:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:00,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:00,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/b21ea09cd8014488b6a48ea17fd7747a is 50, key is test_row_0/A:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:00,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177700914, max timestamp=1733177700924Num uniques 859; 2024-12-02T22:15:00,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:00,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177700914, max timestamp=1733177700924Num uniques 859; 2024-12-02T22:15:00,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177700914, max timestamp=1733177700924Num uniques 859; 2024-12-02T22:15:00,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:00,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:00,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742148_1324 (size=12301) 2024-12-02T22:15:00,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/b21ea09cd8014488b6a48ea17fd7747a 2024-12-02T22:15:00,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/b90061b9d403446085c7af6275cd4757 is 50, key is test_row_0/B:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:00,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177760949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177760952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177760958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177760964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742149_1325 (size=12301) 2024-12-02T22:15:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177760967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177761064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177761073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177761073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177761073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177761083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T22:15:01,153 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:01,154 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-02T22:15:01,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:01,157 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:01,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T22:15:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T22:15:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177761274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177761283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177761284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177761284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177761293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T22:15:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:01,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/b90061b9d403446085c7af6275cd4757 2024-12-02T22:15:01,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5affbfec641644de989144812fd4212c is 50, key is test_row_0/C:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:01,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742150_1326 (size=12301) 2024-12-02T22:15:01,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5affbfec641644de989144812fd4212c 2024-12-02T22:15:01,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/b21ea09cd8014488b6a48ea17fd7747a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a 2024-12-02T22:15:01,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T22:15:01,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/b90061b9d403446085c7af6275cd4757 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757 2024-12-02T22:15:01,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T22:15:01,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5affbfec641644de989144812fd4212c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c 2024-12-02T22:15:01,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T22:15:01,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 545ms, sequenceid=417, compaction requested=true 2024-12-02T22:15:01,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:01,431 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:01,432 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:01,433 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:01,433 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:15:01,433 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,433 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a746a10eefaa4415be8beaab14694873, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.9 K 2024-12-02T22:15:01,433 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:01,433 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:15:01,433 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,434 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8f4729ef89d44c2dbc12f7cf0fb86621, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.9 K 2024-12-02T22:15:01,434 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4729ef89d44c2dbc12f7cf0fb86621, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177697837 2024-12-02T22:15:01,434 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a746a10eefaa4415be8beaab14694873, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177697837 2024-12-02T22:15:01,434 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e12ca5ee15f454a90977d2040586607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733177700093 2024-12-02T22:15:01,435 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 281513b04ff04773b328edf4112e5abd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733177700093 2024-12-02T22:15:01,435 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b90061b9d403446085c7af6275cd4757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:01,435 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21ea09cd8014488b6a48ea17fd7747a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:01,447 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:01,447 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/18f77532533f4facaaa9fd7e725bd28c is 50, key is test_row_0/B:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:01,463 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:01,463 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1ce31d9f0d85421aba5a1a0e6b423eb8 is 50, key is test_row_0/A:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:01,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T22:15:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:01,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T22:15:01,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/fa4ceadd895c44e1a15033e7dae508b0 is 50, key is test_row_0/A:col10/1733177700944/Put/seqid=0 2024-12-02T22:15:01,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742151_1327 (size=13255) 2024-12-02T22:15:01,522 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/18f77532533f4facaaa9fd7e725bd28c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/18f77532533f4facaaa9fd7e725bd28c 2024-12-02T22:15:01,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742152_1328 (size=13255) 2024-12-02T22:15:01,529 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 18f77532533f4facaaa9fd7e725bd28c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:01,529 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:01,530 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177701432; duration=0sec 2024-12-02T22:15:01,530 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:01,530 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:01,530 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:01,531 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:01,531 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:15:01,532 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/1ce31d9f0d85421aba5a1a0e6b423eb8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1ce31d9f0d85421aba5a1a0e6b423eb8 2024-12-02T22:15:01,535 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c6331283c59b422285f0b865b78b3224, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=36.9 K 2024-12-02T22:15:01,535 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c6331283c59b422285f0b865b78b3224, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733177697837 2024-12-02T22:15:01,536 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 50bbce0b049b4c14887d94feeb5e0515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733177700093 2024-12-02T22:15:01,537 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5affbfec641644de989144812fd4212c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:01,541 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into 1ce31d9f0d85421aba5a1a0e6b423eb8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:01,541 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:01,541 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177701431; duration=0sec 2024-12-02T22:15:01,541 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:01,541 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:01,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742153_1329 (size=12301) 2024-12-02T22:15:01,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/fa4ceadd895c44e1a15033e7dae508b0 2024-12-02T22:15:01,575 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:01,575 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7adaefd8fdf6463ea370a004e9fb0812 is 50, key is test_row_0/C:col10/1733177700250/Put/seqid=0 2024-12-02T22:15:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/4b29d0b5210848df9d6270ec92d894c0 is 50, key is test_row_0/B:col10/1733177700944/Put/seqid=0 2024-12-02T22:15:01,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:01,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177761608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177761610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177761613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177761614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177761617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742155_1331 (size=13255) 2024-12-02T22:15:01,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742154_1330 (size=12301) 2024-12-02T22:15:01,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/4b29d0b5210848df9d6270ec92d894c0 2024-12-02T22:15:01,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5212ecba849d4547bd0de9b885fcc6c4 is 50, key is test_row_0/C:col10/1733177700944/Put/seqid=0 2024-12-02T22:15:01,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742156_1332 (size=12301) 2024-12-02T22:15:01,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5212ecba849d4547bd0de9b885fcc6c4 2024-12-02T22:15:01,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/fa4ceadd895c44e1a15033e7dae508b0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0 2024-12-02T22:15:01,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0, entries=150, sequenceid=454, filesize=12.0 K 2024-12-02T22:15:01,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/4b29d0b5210848df9d6270ec92d894c0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0 2024-12-02T22:15:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0, entries=150, sequenceid=454, filesize=12.0 K 2024-12-02T22:15:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/5212ecba849d4547bd0de9b885fcc6c4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4 2024-12-02T22:15:01,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4, entries=150, sequenceid=454, filesize=12.0 K 2024-12-02T22:15:01,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 253ms, sequenceid=454, compaction requested=false 2024-12-02T22:15:01,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:01,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-02T22:15:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-02T22:15:01,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:01,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T22:15:01,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 568 msec 2024-12-02T22:15:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:01,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 574 msec 2024-12-02T22:15:01,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d4ccb30d696b42958ded6706da2f6cbc is 50, key is test_row_0/A:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:01,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742157_1333 (size=14741) 2024-12-02T22:15:01,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733177701758, max timestamp=1733177701772Num uniques 859; 2024-12-02T22:15:01,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T22:15:01,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733177701758, max timestamp=1733177701772Num uniques 859; 2024-12-02T22:15:01,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T22:15:01,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733177701758, max timestamp=1733177701772Num uniques 859; 2024-12-02T22:15:01,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T22:15:01,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d4ccb30d696b42958ded6706da2f6cbc 2024-12-02T22:15:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T22:15:01,784 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:01,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-02T22:15:01,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:01,788 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:01,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:01,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1a320d62114145bbbdb7d15a490ecf86 is 50, key is test_row_0/B:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:01,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177761803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177761806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177761807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177761807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177761809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742158_1334 (size=12301) 2024-12-02T22:15:01,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1a320d62114145bbbdb7d15a490ecf86 2024-12-02T22:15:01,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/58ebc65c6f574efa8e5cc93fc1dd810a is 50, key is test_row_0/C:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742159_1335 (size=12301) 2024-12-02T22:15:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177761914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177761926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177761927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177761927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177761927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:01,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T22:15:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:01,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,033 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7adaefd8fdf6463ea370a004e9fb0812 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7adaefd8fdf6463ea370a004e9fb0812 2024-12-02T22:15:02,039 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 7adaefd8fdf6463ea370a004e9fb0812(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:02,039 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,039 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177701445; duration=0sec 2024-12-02T22:15:02,039 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:02,039 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:02,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T22:15:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177762123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177762143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177762144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177762144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177762144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T22:15:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:02,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/58ebc65c6f574efa8e5cc93fc1dd810a 2024-12-02T22:15:02,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d4ccb30d696b42958ded6706da2f6cbc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc 2024-12-02T22:15:02,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc, entries=200, sequenceid=467, filesize=14.4 K 2024-12-02T22:15:02,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/1a320d62114145bbbdb7d15a490ecf86 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86 2024-12-02T22:15:02,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86, entries=150, sequenceid=467, filesize=12.0 K 2024-12-02T22:15:02,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/58ebc65c6f574efa8e5cc93fc1dd810a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a 2024-12-02T22:15:02,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a, entries=150, sequenceid=467, filesize=12.0 K 2024-12-02T22:15:02,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 604ms, sequenceid=467, compaction requested=true 2024-12-02T22:15:02,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,330 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:02,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:02,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:02,330 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:02,332 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:02,332 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:15:02,332 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,332 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:02,332 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:15:02,332 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/18f77532533f4facaaa9fd7e725bd28c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.0 K 2024-12-02T22:15:02,332 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,333 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1ce31d9f0d85421aba5a1a0e6b423eb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=39.4 K 2024-12-02T22:15:02,333 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f77532533f4facaaa9fd7e725bd28c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:02,333 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce31d9f0d85421aba5a1a0e6b423eb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:02,333 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4ceadd895c44e1a15033e7dae508b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733177700944 2024-12-02T22:15:02,333 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b29d0b5210848df9d6270ec92d894c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733177700944 2024-12-02T22:15:02,334 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ccb30d696b42958ded6706da2f6cbc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:02,334 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a320d62114145bbbdb7d15a490ecf86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:02,342 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:02,342 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:02,343 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/76dd2007f7ca44659d8e0e0419590ab7 is 50, key is test_row_0/B:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:02,343 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/7236302bbb4b4ef3b5174797afd6065c is 50, key is test_row_0/A:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:02,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742161_1337 (size=13357) 2024-12-02T22:15:02,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742160_1336 (size=13357) 2024-12-02T22:15:02,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T22:15:02,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:02,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/903320b99a454675927c0dca56fa9b39 is 50, key is test_row_0/A:col10/1733177701792/Put/seqid=0 2024-12-02T22:15:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742162_1338 (size=12301) 2024-12-02T22:15:02,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177762509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177762510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177762508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177762510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177762519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177762623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177762624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177762624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177762633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177762637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,765 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/7236302bbb4b4ef3b5174797afd6065c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7236302bbb4b4ef3b5174797afd6065c 2024-12-02T22:15:02,767 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/76dd2007f7ca44659d8e0e0419590ab7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/76dd2007f7ca44659d8e0e0419590ab7 2024-12-02T22:15:02,772 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into 7236302bbb4b4ef3b5174797afd6065c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:02,772 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,772 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177702330; duration=0sec 2024-12-02T22:15:02,772 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:02,772 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:02,772 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:02,774 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:02,775 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:15:02,775 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,775 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7adaefd8fdf6463ea370a004e9fb0812, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.0 K 2024-12-02T22:15:02,775 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adaefd8fdf6463ea370a004e9fb0812, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733177700245 2024-12-02T22:15:02,776 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5212ecba849d4547bd0de9b885fcc6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733177700944 2024-12-02T22:15:02,777 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ebc65c6f574efa8e5cc93fc1dd810a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:02,779 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 76dd2007f7ca44659d8e0e0419590ab7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:02,779 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,779 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177702330; duration=0sec 2024-12-02T22:15:02,779 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:02,779 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:02,786 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:02,786 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7548ffdb4d3646048c6ebc6674ac6b5d is 50, key is test_row_0/C:col10/1733177701610/Put/seqid=0 2024-12-02T22:15:02,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742163_1339 (size=13357) 2024-12-02T22:15:02,795 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/7548ffdb4d3646048c6ebc6674ac6b5d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7548ffdb4d3646048c6ebc6674ac6b5d 2024-12-02T22:15:02,800 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 7548ffdb4d3646048c6ebc6674ac6b5d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:02,800 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,800 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177702331; duration=0sec 2024-12-02T22:15:02,800 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:02,800 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:02,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177762833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177762834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177762843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177762843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/903320b99a454675927c0dca56fa9b39 2024-12-02T22:15:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177762853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:02,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/171ea81161f248728714e9f79a2cc65f is 50, key is test_row_0/B:col10/1733177701792/Put/seqid=0 2024-12-02T22:15:02,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742164_1340 (size=12301) 2024-12-02T22:15:02,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/171ea81161f248728714e9f79a2cc65f 2024-12-02T22:15:02,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/24f4aa15384641c9b42592c3037a5843 is 50, key is test_row_0/C:col10/1733177701792/Put/seqid=0 2024-12-02T22:15:02,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742165_1341 (size=12301) 2024-12-02T22:15:02,895 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/24f4aa15384641c9b42592c3037a5843 2024-12-02T22:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/903320b99a454675927c0dca56fa9b39 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39 2024-12-02T22:15:02,906 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T22:15:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/171ea81161f248728714e9f79a2cc65f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f 2024-12-02T22:15:02,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T22:15:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/24f4aa15384641c9b42592c3037a5843 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843 2024-12-02T22:15:02,922 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T22:15:02,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 519ms, sequenceid=502, compaction requested=false 2024-12-02T22:15:02,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:02,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:02,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-02T22:15:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-02T22:15:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:02,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T22:15:02,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1380 sec 2024-12-02T22:15:02,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1440 sec 2024-12-02T22:15:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:03,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T22:15:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:03,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f78f8aa3590b4110a1961d4fdfa5edc2 is 50, key is test_row_0/A:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742166_1342 (size=12301) 2024-12-02T22:15:03,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f78f8aa3590b4110a1961d4fdfa5edc2 2024-12-02T22:15:03,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/d109f121d08e41678badace4b5cf0170 is 50, key is test_row_0/B:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177763175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177763176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177763179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177763180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177763181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742167_1343 (size=12301) 2024-12-02T22:15:03,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/d109f121d08e41678badace4b5cf0170 2024-12-02T22:15:03,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3e463ba90f747b193bb4edd594596d9 is 50, key is test_row_0/C:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742168_1344 (size=12301) 2024-12-02T22:15:03,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3e463ba90f747b193bb4edd594596d9 2024-12-02T22:15:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/f78f8aa3590b4110a1961d4fdfa5edc2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2 2024-12-02T22:15:03,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2, entries=150, sequenceid=518, filesize=12.0 K 2024-12-02T22:15:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/d109f121d08e41678badace4b5cf0170 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170 2024-12-02T22:15:03,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170, entries=150, sequenceid=518, filesize=12.0 K 2024-12-02T22:15:03,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/f3e463ba90f747b193bb4edd594596d9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9 2024-12-02T22:15:03,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9, entries=150, sequenceid=518, filesize=12.0 K 2024-12-02T22:15:03,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 125ms, sequenceid=518, compaction requested=true 2024-12-02T22:15:03,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:03,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:03,270 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:03,271 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:03,271 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:15:03,271 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:03,272 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7236302bbb4b4ef3b5174797afd6065c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.1 K 2024-12-02T22:15:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:03,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:03,272 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7236302bbb4b4ef3b5174797afd6065c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:03,273 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 903320b99a454675927c0dca56fa9b39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733177701792 2024-12-02T22:15:03,273 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78f8aa3590b4110a1961d4fdfa5edc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:03,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:03,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:15:03,273 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:03,274 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/76dd2007f7ca44659d8e0e0419590ab7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.1 K 2024-12-02T22:15:03,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 76dd2007f7ca44659d8e0e0419590ab7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:03,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 171ea81161f248728714e9f79a2cc65f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733177701792 2024-12-02T22:15:03,275 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d109f121d08e41678badace4b5cf0170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:03,283 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:03,283 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:03,283 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/c99c2a57b6654642b7c14be8f22832ea is 50, key is test_row_0/A:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,283 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/beb7c94e93b045b1afa38603f4845555 is 50, key is test_row_0/B:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742169_1345 (size=13459) 2024-12-02T22:15:03,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742170_1346 (size=13459) 2024-12-02T22:15:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9555, totalHeapSize=21976, min timestamp=1733177703173, max timestamp=1733177703300Num uniques 859; 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1733177703173, max timestamp=1733177703300Num uniques 859; 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9557, totalHeapSize=21976, min timestamp=1733177703173, max timestamp=1733177703300Num uniques 859; 2024-12-02T22:15:03,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39327, new segment data size=9557, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39325, new segment data size=9555, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:03,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/6c018130a0f44c80a664da49a3976bda is 50, key is test_row_0/A:col10/1733177703295/Put/seqid=0 2024-12-02T22:15:03,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742171_1347 (size=12301) 2024-12-02T22:15:03,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/6c018130a0f44c80a664da49a3976bda 2024-12-02T22:15:03,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177703366, max timestamp=1733177703381Num uniques 859; 2024-12-02T22:15:03,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:03,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733177703366, max timestamp=1733177703381Num uniques 859; 2024-12-02T22:15:03,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T22:15:03,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733177703366, max timestamp=1733177703381Num uniques 859; 2024-12-02T22:15:03,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T22:15:03,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/a8e4c35c77f74f47b7d313278fcf60a1 is 50, key is test_row_0/B:col10/1733177703295/Put/seqid=0 2024-12-02T22:15:03,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177763411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742172_1348 (size=12301) 2024-12-02T22:15:03,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/a8e4c35c77f74f47b7d313278fcf60a1 2024-12-02T22:15:03,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177763418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177763420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177763420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177763420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/42c9fea284a2485bbc34636478bb2d56 is 50, key is test_row_0/C:col10/1733177703295/Put/seqid=0 2024-12-02T22:15:03,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742173_1349 (size=12301) 2024-12-02T22:15:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177763523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177763533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177763533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177763533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177763533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,703 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/c99c2a57b6654642b7c14be8f22832ea as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c99c2a57b6654642b7c14be8f22832ea 2024-12-02T22:15:03,703 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/beb7c94e93b045b1afa38603f4845555 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/beb7c94e93b045b1afa38603f4845555 2024-12-02T22:15:03,709 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into beb7c94e93b045b1afa38603f4845555(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:03,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:03,709 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177703272; duration=0sec 2024-12-02T22:15:03,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:03,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:03,710 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:03,710 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into c99c2a57b6654642b7c14be8f22832ea(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:03,710 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:03,710 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177703270; duration=0sec 2024-12-02T22:15:03,710 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:03,710 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:03,711 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:03,711 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:15:03,711 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:03,711 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7548ffdb4d3646048c6ebc6674ac6b5d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.1 K 2024-12-02T22:15:03,712 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7548ffdb4d3646048c6ebc6674ac6b5d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1733177701610 2024-12-02T22:15:03,712 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f4aa15384641c9b42592c3037a5843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733177701792 2024-12-02T22:15:03,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e463ba90f747b193bb4edd594596d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177763733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,739 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:03,739 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/55ddd194871441ca854a4a3969c45461 is 50, key is test_row_0/C:col10/1733177703144/Put/seqid=0 2024-12-02T22:15:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177763744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177763744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177763744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177763744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742174_1350 (size=13459) 2024-12-02T22:15:03,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/42c9fea284a2485bbc34636478bb2d56 2024-12-02T22:15:03,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/6c018130a0f44c80a664da49a3976bda as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda 2024-12-02T22:15:03,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda, entries=150, sequenceid=542, filesize=12.0 K 2024-12-02T22:15:03,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/a8e4c35c77f74f47b7d313278fcf60a1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1 2024-12-02T22:15:03,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1, entries=150, sequenceid=542, filesize=12.0 K 2024-12-02T22:15:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/42c9fea284a2485bbc34636478bb2d56 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56 2024-12-02T22:15:03,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56, entries=150, sequenceid=542, filesize=12.0 K 2024-12-02T22:15:03,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.19 KB/144576, currentSize=137.86 KB/141170 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 545ms, sequenceid=542, compaction requested=false 2024-12-02T22:15:03,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T22:15:03,933 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:03,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-02T22:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:03,936 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:03,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:03,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:04,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177703418, max timestamp=1733177704057Num uniques 1118; 2024-12-02T22:15:04,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-02T22:15:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:04,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=108.79 KB heapSize=310.30 KB 2024-12-02T22:15:04,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177703418, max timestamp=1733177704059Num uniques 1009; 2024-12-02T22:15:04,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:04,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177703418, max timestamp=1733177704060Num uniques 1009; 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:04,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d3612c01850845be88c4bcac96e24a0b is 50, key is test_row_0/A:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742175_1351 (size=12301) 2024-12-02T22:15:04,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d3612c01850845be88c4bcac96e24a0b 2024-12-02T22:15:04,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/911f1606dfe742389afebc1a1f546422 is 50, key is test_row_0/B:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T22:15:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177704080, max timestamp=1733177704097Num uniques 859; 2024-12-02T22:15:04,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:04,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177704080, max timestamp=1733177704098Num uniques 859; 2024-12-02T22:15:04,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:04,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177704080, max timestamp=1733177704098Num uniques 859; 2024-12-02T22:15:04,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:04,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742176_1352 (size=12301) 2024-12-02T22:15:04,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/911f1606dfe742389afebc1a1f546422 2024-12-02T22:15:04,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/717fb24a0b8a4a459137f3520190e9f0 is 50, key is test_row_0/C:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177704122, max timestamp=1733177704134Num uniques 1068; 2024-12-02T22:15:04,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177704122, max timestamp=1733177704134Num uniques 1068; 2024-12-02T22:15:04,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:04,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:04,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177704122, max timestamp=1733177704134Num uniques 1068; 2024-12-02T22:15:04,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:04,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742177_1353 (size=12301) 2024-12-02T22:15:04,154 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/55ddd194871441ca854a4a3969c45461 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/55ddd194871441ca854a4a3969c45461 2024-12-02T22:15:04,161 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 55ddd194871441ca854a4a3969c45461(size=13.1 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:04,161 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:04,161 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177703272; duration=0sec 2024-12-02T22:15:04,161 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:04,161 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:04,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733177704160, max timestamp=1733177704173Num uniques 1027; 2024-12-02T22:15:04,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733177704160, max timestamp=1733177704173Num uniques 1027; 2024-12-02T22:15:04,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T22:15:04,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T22:15:04,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733177704160, max timestamp=1733177704173Num uniques 1027; 2024-12-02T22:15:04,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T22:15:04,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177764198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177764200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177764200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177764202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177764203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T22:15:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:04,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177764313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177764313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177764313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177764313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177764313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T22:15:04,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177764523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177764524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177764524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177764524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177764524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/717fb24a0b8a4a459137f3520190e9f0 2024-12-02T22:15:04,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/d3612c01850845be88c4bcac96e24a0b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b 2024-12-02T22:15:04,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T22:15:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:04,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b, entries=150, sequenceid=584, filesize=12.0 K 2024-12-02T22:15:04,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/911f1606dfe742389afebc1a1f546422 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422 2024-12-02T22:15:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:04,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422, entries=150, sequenceid=584, filesize=12.0 K 2024-12-02T22:15:04,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/717fb24a0b8a4a459137f3520190e9f0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0 2024-12-02T22:15:04,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0, entries=150, sequenceid=584, filesize=12.0 K 2024-12-02T22:15:04,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 510ms, sequenceid=584, compaction requested=true 2024-12-02T22:15:04,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:04,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:04,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:04,572 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:04,573 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:04,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:04,573 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:04,573 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/A is initiating minor compaction (all files) 2024-12-02T22:15:04,573 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/A in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,574 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:04,574 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/B is initiating minor compaction (all files) 2024-12-02T22:15:04,574 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c99c2a57b6654642b7c14be8f22832ea, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.2 K 2024-12-02T22:15:04,574 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/B in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,574 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/beb7c94e93b045b1afa38603f4845555, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.2 K 2024-12-02T22:15:04,574 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99c2a57b6654642b7c14be8f22832ea, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:04,574 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting beb7c94e93b045b1afa38603f4845555, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:04,575 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c018130a0f44c80a664da49a3976bda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733177703286 2024-12-02T22:15:04,575 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e4c35c77f74f47b7d313278fcf60a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733177703286 2024-12-02T22:15:04,575 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3612c01850845be88c4bcac96e24a0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733177703418 2024-12-02T22:15:04,575 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 911f1606dfe742389afebc1a1f546422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733177703418 2024-12-02T22:15:04,585 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#A#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:04,585 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:04,585 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/e2bbdbb14c29463abfd2b846226c04c9 is 50, key is test_row_0/A:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8a4d2a6feb9f44db973fff37eb1c254b is 50, key is test_row_0/B:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742179_1355 (size=13561) 2024-12-02T22:15:04,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742178_1354 (size=13561) 2024-12-02T22:15:04,597 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/8a4d2a6feb9f44db973fff37eb1c254b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8a4d2a6feb9f44db973fff37eb1c254b 2024-12-02T22:15:04,602 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/B of b71ff10c7b688fa2fccab1c4c47e3ef0 into 8a4d2a6feb9f44db973fff37eb1c254b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:04,602 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:04,602 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/B, priority=13, startTime=1733177704572; duration=0sec 2024-12-02T22:15:04,602 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:04,602 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:04,602 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:04,605 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:04,605 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): b71ff10c7b688fa2fccab1c4c47e3ef0/C is initiating minor compaction (all files) 2024-12-02T22:15:04,605 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b71ff10c7b688fa2fccab1c4c47e3ef0/C in TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,605 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/55ddd194871441ca854a4a3969c45461, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp, totalSize=37.2 K 2024-12-02T22:15:04,605 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 55ddd194871441ca854a4a3969c45461, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733177702466 2024-12-02T22:15:04,605 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c9fea284a2485bbc34636478bb2d56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733177703286 2024-12-02T22:15:04,606 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 717fb24a0b8a4a459137f3520190e9f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1733177703418 2024-12-02T22:15:04,613 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b71ff10c7b688fa2fccab1c4c47e3ef0#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:04,614 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/553a52072a9b4dfc87deff6d386f1f34 is 50, key is test_row_0/C:col10/1733177704059/Put/seqid=0 2024-12-02T22:15:04,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742180_1356 (size=13561) 2024-12-02T22:15:04,636 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/553a52072a9b4dfc87deff6d386f1f34 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/553a52072a9b4dfc87deff6d386f1f34 2024-12-02T22:15:04,644 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/C of b71ff10c7b688fa2fccab1c4c47e3ef0 into 553a52072a9b4dfc87deff6d386f1f34(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:04,644 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:04,644 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/C, priority=13, startTime=1733177704573; duration=0sec 2024-12-02T22:15:04,644 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:04,644 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:04,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T22:15:04,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:04,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=147.60 KB heapSize=409.08 KB 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:04,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8e74eba55e5c485daf74b3480ed882ea is 50, key is test_row_0/A:col10/1733177704202/Put/seqid=0 2024-12-02T22:15:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742181_1357 (size=12301) 2024-12-02T22:15:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:04,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. as already flushing 2024-12-02T22:15:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177764839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177764840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177764840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177764844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177764846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177764953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177764953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177764953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177764953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177764953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,002 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/e2bbdbb14c29463abfd2b846226c04c9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e2bbdbb14c29463abfd2b846226c04c9 2024-12-02T22:15:05,010 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b71ff10c7b688fa2fccab1c4c47e3ef0/A of b71ff10c7b688fa2fccab1c4c47e3ef0 into e2bbdbb14c29463abfd2b846226c04c9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:05,011 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:05,011 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., storeName=b71ff10c7b688fa2fccab1c4c47e3ef0/A, priority=13, startTime=1733177704572; duration=0sec 2024-12-02T22:15:05,011 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:05,011 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:05,124 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8e74eba55e5c485daf74b3480ed882ea 2024-12-02T22:15:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c82d84c604fb43c381db519533f435a4 is 50, key is test_row_0/B:col10/1733177704202/Put/seqid=0 2024-12-02T22:15:05,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742182_1358 (size=12301) 2024-12-02T22:15:05,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177765474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177765474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177765474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177765474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177765484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c82d84c604fb43c381db519533f435a4 2024-12-02T22:15:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/b05965da317b48129667703676a6b11f is 50, key is test_row_0/C:col10/1733177704202/Put/seqid=0 2024-12-02T22:15:05,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742183_1359 (size=12301) 2024-12-02T22:15:05,981 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-12-02T22:15:05,981 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-12-02T22:15:05,981 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:05,981 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:05,981 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,981 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,981 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,981 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:05,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:05,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60540 deadline: 1733177765983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60546 deadline: 1733177765983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,985 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-12-02T22:15:05,985 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,985 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,985 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,985 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-12-02T22:15:05,985 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:05,985 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,985 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:05,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:05,987 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-12-02T22:15:05,987 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:05,987 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,987 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:05,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60472 deadline: 1733177765993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1733177765993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:06,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/b05965da317b48129667703676a6b11f 2024-12-02T22:15:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733177766003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8e74eba55e5c485daf74b3480ed882ea as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8e74eba55e5c485daf74b3480ed882ea 2024-12-02T22:15:06,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8e74eba55e5c485daf74b3480ed882ea, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T22:15:06,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/c82d84c604fb43c381db519533f435a4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c82d84c604fb43c381db519533f435a4 2024-12-02T22:15:06,019 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c82d84c604fb43c381db519533f435a4, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T22:15:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/b05965da317b48129667703676a6b11f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/b05965da317b48129667703676a6b11f 2024-12-02T22:15:06,023 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/b05965da317b48129667703676a6b11f, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T22:15:06,024 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~409.03 KB/418848, currentSize=46.96 KB/48090 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 1315ms, sequenceid=657, compaction requested=false 2024-12-02T22:15:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-02T22:15:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-02T22:15:06,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-02T22:15:06,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0880 sec 2024-12-02T22:15:06,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0930 sec 2024-12-02T22:15:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T22:15:06,083 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:06,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:15:06,995 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-12-02T22:15:06,995 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:06,995 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:06,995 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:06,995 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-12-02T22:15:06,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:06,995 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:06,995 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:06,995 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:06,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:06,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/24cb4baaa7ac4d9ab60ae7567533a2ac is 50, key is test_row_0/A:col10/1733177706993/Put/seqid=0 2024-12-02T22:15:07,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742184_1360 (size=12301) 2024-12-02T22:15:07,005 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-12-02T22:15:07,005 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-12-02T22:15:07,005 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:07,005 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,005 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:07,005 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,005 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:07,005 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:07,015 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-12-02T22:15:07,015 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:07,015 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,015 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2431 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2416 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2386 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2445 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2421 2024-12-02T22:15:07,015 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:15:07,016 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:15:07,016 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:15:07,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:07,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:07,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:15:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:07,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177707019"}]},"ts":"1733177707019"} 2024-12-02T22:15:07,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:15:07,020 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:15:07,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:15:07,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, UNASSIGN}] 2024-12-02T22:15:07,023 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, UNASSIGN 2024-12-02T22:15:07,024 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b71ff10c7b688fa2fccab1c4c47e3ef0, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:07,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, UNASSIGN because future has completed 2024-12-02T22:15:07,026 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:15:07,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:07,180 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:07,180 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:15:07,180 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing b71ff10c7b688fa2fccab1c4c47e3ef0, disabling compactions & flushes 2024-12-02T22:15:07,180 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:07,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/24cb4baaa7ac4d9ab60ae7567533a2ac 2024-12-02T22:15:07,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/953aeb8ee2f04717b34ee6770cdd93b2 is 50, key is test_row_0/B:col10/1733177706993/Put/seqid=0 2024-12-02T22:15:07,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742185_1361 (size=12301) 2024-12-02T22:15:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:07,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/953aeb8ee2f04717b34ee6770cdd93b2 2024-12-02T22:15:07,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/d9ac968202db40a2ac196e6b47e6af77 is 50, key is test_row_0/C:col10/1733177706993/Put/seqid=0 2024-12-02T22:15:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742186_1362 (size=12301) 2024-12-02T22:15:07,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/d9ac968202db40a2ac196e6b47e6af77 2024-12-02T22:15:07,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/24cb4baaa7ac4d9ab60ae7567533a2ac as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/24cb4baaa7ac4d9ab60ae7567533a2ac 2024-12-02T22:15:07,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/24cb4baaa7ac4d9ab60ae7567533a2ac, entries=150, sequenceid=670, filesize=12.0 K 2024-12-02T22:15:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/953aeb8ee2f04717b34ee6770cdd93b2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/953aeb8ee2f04717b34ee6770cdd93b2 2024-12-02T22:15:07,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/953aeb8ee2f04717b34ee6770cdd93b2, entries=150, sequenceid=670, filesize=12.0 K 2024-12-02T22:15:07,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/d9ac968202db40a2ac196e6b47e6af77 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d9ac968202db40a2ac196e6b47e6af77 2024-12-02T22:15:07,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d9ac968202db40a2ac196e6b47e6af77, entries=150, sequenceid=670, filesize=12.0 K 2024-12-02T22:15:07,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 847ms, sequenceid=670, compaction requested=true 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b71ff10c7b688fa2fccab1c4c47e3ef0: 2024-12-02T22:15:07,841 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. after waiting 0 ms 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:A 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b71ff10c7b688fa2fccab1c4c47e3ef0:C, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:07,841 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing b71ff10c7b688fa2fccab1c4c47e3ef0 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. because compaction request was cancelled 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:B 2024-12-02T22:15:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:07,841 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b71ff10c7b688fa2fccab1c4c47e3ef0:C 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=A 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=B 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b71ff10c7b688fa2fccab1c4c47e3ef0, store=C 2024-12-02T22:15:07,841 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:07,845 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8ffbde32710e41ccb4d483fadff32f62 is 50, key is test_row_0/A:col10/1733177707003/Put/seqid=0 2024-12-02T22:15:07,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742187_1363 (size=9857) 2024-12-02T22:15:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:08,257 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8ffbde32710e41ccb4d483fadff32f62 2024-12-02T22:15:08,263 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3a5bb2e62170440a8ab469a2cdd349ee is 50, key is test_row_0/B:col10/1733177707003/Put/seqid=0 2024-12-02T22:15:08,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742188_1364 (size=9857) 2024-12-02T22:15:08,267 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3a5bb2e62170440a8ab469a2cdd349ee 2024-12-02T22:15:08,274 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/4ff848102cf34672864a571ccc3412c4 is 50, key is test_row_0/C:col10/1733177707003/Put/seqid=0 2024-12-02T22:15:08,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742189_1365 (size=9857) 2024-12-02T22:15:08,677 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/4ff848102cf34672864a571ccc3412c4 2024-12-02T22:15:08,681 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/A/8ffbde32710e41ccb4d483fadff32f62 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8ffbde32710e41ccb4d483fadff32f62 2024-12-02T22:15:08,684 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8ffbde32710e41ccb4d483fadff32f62, entries=100, sequenceid=676, filesize=9.6 K 2024-12-02T22:15:08,685 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/B/3a5bb2e62170440a8ab469a2cdd349ee as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3a5bb2e62170440a8ab469a2cdd349ee 2024-12-02T22:15:08,688 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3a5bb2e62170440a8ab469a2cdd349ee, entries=100, sequenceid=676, filesize=9.6 K 2024-12-02T22:15:08,689 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/.tmp/C/4ff848102cf34672864a571ccc3412c4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/4ff848102cf34672864a571ccc3412c4 2024-12-02T22:15:08,692 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/4ff848102cf34672864a571ccc3412c4, entries=100, sequenceid=676, filesize=9.6 K 2024-12-02T22:15:08,692 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 851ms, sequenceid=676, compaction requested=true 2024-12-02T22:15:08,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/786d84fd4de24a0d8f22b361840757c6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1634e7a64e244bae9a7e065248f3e659, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f34733765af3450c97c294c9a7fcb013, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b038fa9ad35a42dabe2f08a99eeb6d3e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/cf98a4bbc7604886b844ae4393243113, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/654e669f82754587a9cd127d3e1fc98a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a746a10eefaa4415be8beaab14694873, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1ce31d9f0d85421aba5a1a0e6b423eb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7236302bbb4b4ef3b5174797afd6065c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c99c2a57b6654642b7c14be8f22832ea, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b] to archive 2024-12-02T22:15:08,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:08,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8054d5a824ba4158b78aa2bc6c96f97a 2024-12-02T22:15:08,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a19a2a90db534514b2a04ce4a303ae0e 2024-12-02T22:15:08,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e6d49d6a2aac4f6e952ad3a96680f56b 2024-12-02T22:15:08,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff1ffff968e749fab29964bf0f7b53a8 2024-12-02T22:15:08,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/786d84fd4de24a0d8f22b361840757c6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/786d84fd4de24a0d8f22b361840757c6 2024-12-02T22:15:08,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/3948884a0a384d3db76d0c9c461942b9 2024-12-02T22:15:08,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/44bea61e191145ad903b6f3142029685 2024-12-02T22:15:08,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1634e7a64e244bae9a7e065248f3e659 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1634e7a64e244bae9a7e065248f3e659 2024-12-02T22:15:08,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/ff3d7f0ecc934502bd649b7fa2602e7a 2024-12-02T22:15:08,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/978383e7cf934f70985aec7e7730e942 2024-12-02T22:15:08,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0f5db566e1d94f42990fd2003dc6421d 2024-12-02T22:15:08,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f34733765af3450c97c294c9a7fcb013 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f34733765af3450c97c294c9a7fcb013 2024-12-02T22:15:08,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c9bfecb6b6ba4dc4944275bc4c5cb002 2024-12-02T22:15:08,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b038fa9ad35a42dabe2f08a99eeb6d3e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b038fa9ad35a42dabe2f08a99eeb6d3e 2024-12-02T22:15:08,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f28900b7def54c2c94734c0ecdefe732 2024-12-02T22:15:08,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/38cf546f96a1475086ec7b6b1e950bd6 2024-12-02T22:15:08,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7b7365fdbd254960ad57502b546a7c13 2024-12-02T22:15:08,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/cf98a4bbc7604886b844ae4393243113 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/cf98a4bbc7604886b844ae4393243113 2024-12-02T22:15:08,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/148c78af3cb64b1fa36d46ff5bb98593 2024-12-02T22:15:08,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1b5eab5ed17a429fa991c9586362fe51 2024-12-02T22:15:08,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/821d767889014038b4de22b5d10f14bc 2024-12-02T22:15:08,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/5c0b80dc35b3413f9b92a5b56df66b30 2024-12-02T22:15:08,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/654e669f82754587a9cd127d3e1fc98a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/654e669f82754587a9cd127d3e1fc98a 2024-12-02T22:15:08,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/428776b26175426ba43922f420b1b42d 2024-12-02T22:15:08,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a746a10eefaa4415be8beaab14694873 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/a746a10eefaa4415be8beaab14694873 2024-12-02T22:15:08,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/0c39e30f6faf4cbf85329d6fcf709aa7 2024-12-02T22:15:08,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/281513b04ff04773b328edf4112e5abd 2024-12-02T22:15:08,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1ce31d9f0d85421aba5a1a0e6b423eb8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/1ce31d9f0d85421aba5a1a0e6b423eb8 2024-12-02T22:15:08,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/b21ea09cd8014488b6a48ea17fd7747a 2024-12-02T22:15:08,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/fa4ceadd895c44e1a15033e7dae508b0 2024-12-02T22:15:08,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d4ccb30d696b42958ded6706da2f6cbc 2024-12-02T22:15:08,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7236302bbb4b4ef3b5174797afd6065c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/7236302bbb4b4ef3b5174797afd6065c 2024-12-02T22:15:08,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/903320b99a454675927c0dca56fa9b39 2024-12-02T22:15:08,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c99c2a57b6654642b7c14be8f22832ea to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/c99c2a57b6654642b7c14be8f22832ea 2024-12-02T22:15:08,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/f78f8aa3590b4110a1961d4fdfa5edc2 2024-12-02T22:15:08,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/6c018130a0f44c80a664da49a3976bda 2024-12-02T22:15:08,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/d3612c01850845be88c4bcac96e24a0b 2024-12-02T22:15:08,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/0692a3263c994ee799b6c9e58178e41d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/da779b57a0874298b6cd09b59ec1731b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3fb02e0cb1dd499ba8cc4d426177a37f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/cf7539b102a24885af9d1b75beb3e1d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/402d1844d88744a4be5efe1e90a4b831, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/dc679ace3e6544cfb11a4645862c76dd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8f4729ef89d44c2dbc12f7cf0fb86621, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/18f77532533f4facaaa9fd7e725bd28c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/76dd2007f7ca44659d8e0e0419590ab7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/beb7c94e93b045b1afa38603f4845555, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422] to archive 2024-12-02T22:15:08,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:08,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1b8e606b38a547d584a1183c4e25fb4c 2024-12-02T22:15:08,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/6cb92972607246c6907fd3a9169d5408 2024-12-02T22:15:08,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/81fad6c39e7f4986813f78bb6b56833f 2024-12-02T22:15:08,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/0692a3263c994ee799b6c9e58178e41d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/0692a3263c994ee799b6c9e58178e41d 2024-12-02T22:15:08,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2a2d1412d4f744c7a2ca8ce300ee5731 2024-12-02T22:15:08,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e958e25eed5c41aaa853971a93e41e36 2024-12-02T22:15:08,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/edadc30363c74516833c9d4dcaa4b883 2024-12-02T22:15:08,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/da779b57a0874298b6cd09b59ec1731b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/da779b57a0874298b6cd09b59ec1731b 2024-12-02T22:15:08,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c8737c17278246c18787ada57372246b 2024-12-02T22:15:08,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c2d7f27da9b344089191e8f73adf88ee 2024-12-02T22:15:08,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3fb02e0cb1dd499ba8cc4d426177a37f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3fb02e0cb1dd499ba8cc4d426177a37f 2024-12-02T22:15:08,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8c5878d2d4a944e8af92a2229458fcb0 2024-12-02T22:15:08,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/848ace1671ae4385b8f0d2eb133df20c 2024-12-02T22:15:08,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/cf7539b102a24885af9d1b75beb3e1d8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/cf7539b102a24885af9d1b75beb3e1d8 2024-12-02T22:15:08,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3d4a86d40539401ca70e1d7cdcb96aa5 2024-12-02T22:15:08,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/22126585d83646ecb29f47bb47e808da 2024-12-02T22:15:08,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/f02b7be47d544251a2911aab368b2aad 2024-12-02T22:15:08,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/402d1844d88744a4be5efe1e90a4b831 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/402d1844d88744a4be5efe1e90a4b831 2024-12-02T22:15:08,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/795a29e5ef524d15a1522a4550b202e2 2024-12-02T22:15:08,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/98daff4d6da849a997903e061aacf346 2024-12-02T22:15:08,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/e4c3038788fa439a86e654caf431a9e7 2024-12-02T22:15:08,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/dc679ace3e6544cfb11a4645862c76dd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/dc679ace3e6544cfb11a4645862c76dd 2024-12-02T22:15:08,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/2c4073a008cc4f1c9619834a13249b00 2024-12-02T22:15:08,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/df213d3c0a6f4166a91d1be5fb9e3e4e 2024-12-02T22:15:08,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8f4729ef89d44c2dbc12f7cf0fb86621 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8f4729ef89d44c2dbc12f7cf0fb86621 2024-12-02T22:15:08,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/9449d51651bd44bb858b29a081781b96 2024-12-02T22:15:08,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1e12ca5ee15f454a90977d2040586607 2024-12-02T22:15:08,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/18f77532533f4facaaa9fd7e725bd28c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/18f77532533f4facaaa9fd7e725bd28c 2024-12-02T22:15:08,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/b90061b9d403446085c7af6275cd4757 2024-12-02T22:15:08,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/4b29d0b5210848df9d6270ec92d894c0 2024-12-02T22:15:08,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/76dd2007f7ca44659d8e0e0419590ab7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/76dd2007f7ca44659d8e0e0419590ab7 2024-12-02T22:15:08,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/1a320d62114145bbbdb7d15a490ecf86 2024-12-02T22:15:08,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/171ea81161f248728714e9f79a2cc65f 2024-12-02T22:15:08,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/beb7c94e93b045b1afa38603f4845555 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/beb7c94e93b045b1afa38603f4845555 2024-12-02T22:15:08,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/d109f121d08e41678badace4b5cf0170 2024-12-02T22:15:08,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/a8e4c35c77f74f47b7d313278fcf60a1 2024-12-02T22:15:08,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/911f1606dfe742389afebc1a1f546422 2024-12-02T22:15:08,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cf0bb7d2cf2a41daa02ac2b672f1de44, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/043f5ec4300a409aa01dd0fad10ffb04, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/9d03c7ba95ec42e798a7796cacdc0f37, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cc5af2aac19641619ec1240693cf883b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/98760e799bda4226a4c212ca8883582c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d26c51da3ba4464cbd61b3d49b51e58a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c6331283c59b422285f0b865b78b3224, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7adaefd8fdf6463ea370a004e9fb0812, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7548ffdb4d3646048c6ebc6674ac6b5d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/55ddd194871441ca854a4a3969c45461, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0] to archive 2024-12-02T22:15:08,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:08,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e1f0d32bd72b43c3b6b7f0a090cf0d82 2024-12-02T22:15:08,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/414dfbf833c84af1826bd5f72bcbdc78 2024-12-02T22:15:08,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0c1c313e53ec43daa98bc041daaca87d 2024-12-02T22:15:08,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cf0bb7d2cf2a41daa02ac2b672f1de44 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cf0bb7d2cf2a41daa02ac2b672f1de44 2024-12-02T22:15:08,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/830a9f0cb328471a8b19999968f58e5e 2024-12-02T22:15:08,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/51f9e39224924e5fb33cf3085a524e16 2024-12-02T22:15:08,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c3400c03c1d440b5a84bd4968fb4a6fb 2024-12-02T22:15:08,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/043f5ec4300a409aa01dd0fad10ffb04 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/043f5ec4300a409aa01dd0fad10ffb04 2024-12-02T22:15:08,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/e8328fff9a4e4bc58bbbfa561b24e28e 2024-12-02T22:15:08,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/be24c105426947f7948c0448994650d2 2024-12-02T22:15:08,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/9d03c7ba95ec42e798a7796cacdc0f37 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/9d03c7ba95ec42e798a7796cacdc0f37 2024-12-02T22:15:08,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/da67e727ce1d48c2a380dc92f57cd16e 2024-12-02T22:15:08,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3200cc5205a46579a42d1bef802b83b 2024-12-02T22:15:08,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cc5af2aac19641619ec1240693cf883b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/cc5af2aac19641619ec1240693cf883b 2024-12-02T22:15:08,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/632f2c2070d44b87a8745afcac62f21b 2024-12-02T22:15:08,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7579a45db77b4cde9a5ecf077143a766 2024-12-02T22:15:08,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/a38e23e493284d54b6f67242ffd1eb76 2024-12-02T22:15:08,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f86cb1970c0f48e6a41293137d44e391 2024-12-02T22:15:08,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/98760e799bda4226a4c212ca8883582c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/98760e799bda4226a4c212ca8883582c 2024-12-02T22:15:08,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/186f22f68e67495d9a03ad512b906742 2024-12-02T22:15:08,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/0583e1f053704d4eb94c5ba76f31890c 2024-12-02T22:15:08,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d26c51da3ba4464cbd61b3d49b51e58a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d26c51da3ba4464cbd61b3d49b51e58a 2024-12-02T22:15:08,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f270a83035954a4ba1bcc08f60e1c7f8 2024-12-02T22:15:08,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c5f8e9f85cb2489ea4241c25e0efea4d 2024-12-02T22:15:08,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c6331283c59b422285f0b865b78b3224 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/c6331283c59b422285f0b865b78b3224 2024-12-02T22:15:08,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/db02a8b75d424491a877d9037b26a03a 2024-12-02T22:15:08,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/50bbce0b049b4c14887d94feeb5e0515 2024-12-02T22:15:08,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7adaefd8fdf6463ea370a004e9fb0812 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7adaefd8fdf6463ea370a004e9fb0812 2024-12-02T22:15:08,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5affbfec641644de989144812fd4212c 2024-12-02T22:15:08,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/5212ecba849d4547bd0de9b885fcc6c4 2024-12-02T22:15:08,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7548ffdb4d3646048c6ebc6674ac6b5d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/7548ffdb4d3646048c6ebc6674ac6b5d 2024-12-02T22:15:08,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/58ebc65c6f574efa8e5cc93fc1dd810a 2024-12-02T22:15:08,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/24f4aa15384641c9b42592c3037a5843 2024-12-02T22:15:08,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/55ddd194871441ca854a4a3969c45461 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/55ddd194871441ca854a4a3969c45461 2024-12-02T22:15:08,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/f3e463ba90f747b193bb4edd594596d9 2024-12-02T22:15:08,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/42c9fea284a2485bbc34636478bb2d56 2024-12-02T22:15:08,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/717fb24a0b8a4a459137f3520190e9f0 2024-12-02T22:15:08,799 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/recovered.edits/679.seqid, newMaxSeqId=679, maxSeqId=1 2024-12-02T22:15:08,800 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. 2024-12-02T22:15:08,800 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for b71ff10c7b688fa2fccab1c4c47e3ef0: Waiting for close lock at 1733177707180Running coprocessor pre-close hooks at 1733177707180Disabling compacts and flushes for region at 1733177707180Disabling writes for close at 1733177707841 (+661 ms)Obtaining lock to block concurrent updates at 1733177707841Preparing flush snapshotting stores in b71ff10c7b688fa2fccab1c4c47e3ef0 at 1733177707841Finished memstore snapshotting TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733177707841Flushing stores of TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0. at 1733177707842 (+1 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/A: creating writer at 1733177707842Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/A: appending metadata at 1733177707845 (+3 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/A: closing flushed file at 1733177707845Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/B: creating writer at 1733177708261 (+416 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/B: appending metadata at 1733177708263 (+2 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/B: closing flushed file at 1733177708263Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/C: creating writer at 1733177708270 (+7 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/C: appending metadata at 1733177708273 (+3 ms)Flushing b71ff10c7b688fa2fccab1c4c47e3ef0/C: closing flushed file at 1733177708273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7567375c: reopening flushed file at 1733177708681 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@773a0a94: reopening flushed file at 1733177708685 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326ed5e: reopening flushed file at 1733177708688 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for b71ff10c7b688fa2fccab1c4c47e3ef0 in 851ms, sequenceid=676, compaction requested=true at 1733177708692 (+4 ms)Writing region close event to WAL at 1733177708797 (+105 ms)Running coprocessor post-close hooks at 1733177708800 (+3 ms)Closed at 1733177708800 2024-12-02T22:15:08,802 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:08,802 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b71ff10c7b688fa2fccab1c4c47e3ef0, regionState=CLOSED 2024-12-02T22:15:08,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:08,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T22:15:08,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure b71ff10c7b688fa2fccab1c4c47e3ef0, server=7ae73638d47e,41753,1733177627056 in 1.7780 sec 2024-12-02T22:15:08,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-02T22:15:08,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b71ff10c7b688fa2fccab1c4c47e3ef0, UNASSIGN in 1.7830 sec 2024-12-02T22:15:08,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-02T22:15:08,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7870 sec 2024-12-02T22:15:08,811 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177708810"}]},"ts":"1733177708810"} 2024-12-02T22:15:08,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:15:08,812 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:15:08,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7950 sec 2024-12-02T22:15:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T22:15:09,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:09,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:15:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,176 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T22:15:09,177 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,180 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:09,182 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/recovered.edits] 2024-12-02T22:15:09,184 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/24cb4baaa7ac4d9ab60ae7567533a2ac to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/24cb4baaa7ac4d9ab60ae7567533a2ac 2024-12-02T22:15:09,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8e74eba55e5c485daf74b3480ed882ea to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8e74eba55e5c485daf74b3480ed882ea 2024-12-02T22:15:09,186 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8ffbde32710e41ccb4d483fadff32f62 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/8ffbde32710e41ccb4d483fadff32f62 2024-12-02T22:15:09,187 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e2bbdbb14c29463abfd2b846226c04c9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/A/e2bbdbb14c29463abfd2b846226c04c9 2024-12-02T22:15:09,189 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3a5bb2e62170440a8ab469a2cdd349ee to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/3a5bb2e62170440a8ab469a2cdd349ee 2024-12-02T22:15:09,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8a4d2a6feb9f44db973fff37eb1c254b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/8a4d2a6feb9f44db973fff37eb1c254b 2024-12-02T22:15:09,191 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/953aeb8ee2f04717b34ee6770cdd93b2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/953aeb8ee2f04717b34ee6770cdd93b2 2024-12-02T22:15:09,192 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c82d84c604fb43c381db519533f435a4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/B/c82d84c604fb43c381db519533f435a4 2024-12-02T22:15:09,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/4ff848102cf34672864a571ccc3412c4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/4ff848102cf34672864a571ccc3412c4 2024-12-02T22:15:09,195 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/553a52072a9b4dfc87deff6d386f1f34 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/553a52072a9b4dfc87deff6d386f1f34 2024-12-02T22:15:09,196 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/b05965da317b48129667703676a6b11f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/b05965da317b48129667703676a6b11f 2024-12-02T22:15:09,197 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d9ac968202db40a2ac196e6b47e6af77 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/C/d9ac968202db40a2ac196e6b47e6af77 2024-12-02T22:15:09,199 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/recovered.edits/679.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0/recovered.edits/679.seqid 2024-12-02T22:15:09,200 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/b71ff10c7b688fa2fccab1c4c47e3ef0 2024-12-02T22:15:09,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:15:09,202 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,204 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:15:09,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:15:09,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:15:09,207 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177709207"}]},"ts":"9223372036854775807"} 2024-12-02T22:15:09,209 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:15:09,209 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b71ff10c7b688fa2fccab1c4c47e3ef0, NAME => 'TestAcidGuarantees,,1733177684765.b71ff10c7b688fa2fccab1c4c47e3ef0.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:15:09,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:15:09,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177709209"}]},"ts":"9223372036854775807"} 2024-12-02T22:15:09,210 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:15:09,211 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-12-02T22:15:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T22:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:15:09,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:09,293 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 252), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=596 (was 523) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7916 (was 8022) 2024-12-02T22:15:09,302 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=596, ProcessCount=11, AvailableMemoryMB=7916 2024-12-02T22:15:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:15:09,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:15:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:09,306 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:15:09,306 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:09,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-02T22:15:09,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:09,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742190_1366 (size=1061) 2024-12-02T22:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:09,714 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:15:09,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742191_1367 (size=53) 2024-12-02T22:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 87f06add23a346902905707af25d359f, disabling compactions & flushes 2024-12-02T22:15:10,119 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. after waiting 0 ms 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,119 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,119 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87f06add23a346902905707af25d359f: Waiting for close lock at 1733177710119Disabling compacts and flushes for region at 1733177710119Disabling writes for close at 1733177710119Writing region close event to WAL at 1733177710119Closed at 1733177710119 2024-12-02T22:15:10,120 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:15:10,121 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177710120"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177710120"}]},"ts":"1733177710120"} 2024-12-02T22:15:10,122 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:15:10,122 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:15:10,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177710122"}]},"ts":"1733177710122"} 2024-12-02T22:15:10,123 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:15:10,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, ASSIGN}] 2024-12-02T22:15:10,125 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, ASSIGN 2024-12-02T22:15:10,125 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:15:10,276 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:10,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, ASSIGN because future has completed 2024-12-02T22:15:10,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:10,431 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,431 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:15:10,432 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,432 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:15:10,432 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,432 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,433 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,434 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:10,434 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName A 2024-12-02T22:15:10,434 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:10,435 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:10,435 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,435 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:10,436 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName B 2024-12-02T22:15:10,436 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:10,436 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:10,436 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,437 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:10,437 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName C 2024-12-02T22:15:10,437 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:10,437 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:10,437 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,438 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,438 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:10,438 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:10,439 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,439 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,439 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:15:10,440 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,442 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:15:10,442 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 87f06add23a346902905707af25d359f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60487539, jitterRate=-0.09866543114185333}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:15:10,442 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87f06add23a346902905707af25d359f 2024-12-02T22:15:10,442 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 87f06add23a346902905707af25d359f: Running coprocessor pre-open hook at 1733177710432Writing region info on filesystem at 1733177710432Initializing all the Stores at 1733177710432Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177710432Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177710433 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177710433Cleaning up temporary data from old regions at 1733177710439 (+6 ms)Running coprocessor post-open hooks at 1733177710442 (+3 ms)Region opened successfully at 1733177710442 2024-12-02T22:15:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:10,443 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., pid=93, masterSystemTime=1733177710428 2024-12-02T22:15:10,445 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,445 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:10,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:10,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:10,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-02T22:15:10,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 in 171 msec 2024-12-02T22:15:10,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-02T22:15:10,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, ASSIGN in 325 msec 2024-12-02T22:15:10,452 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:15:10,452 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177710452"}]},"ts":"1733177710452"} 2024-12-02T22:15:10,453 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:15:10,454 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:15:10,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-12-02T22:15:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T22:15:11,463 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:11,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:11,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:11,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:11,465 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c4a4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:11,466 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309c0dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:11,467 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:11,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:11,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:15:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:15:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@31819568 2024-12-02T22:15:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:15:11,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:15:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:15:11,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:15:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:11,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742192_1368 (size=1097) 2024-12-02T22:15:11,884 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T22:15:11,884 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T22:15:11,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:15:11,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE}] 2024-12-02T22:15:11,889 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE 2024-12-02T22:15:11,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:11,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE because future has completed 2024-12-02T22:15:11,891 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:15:11,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:12,043 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,043 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:15:12,043 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 87f06add23a346902905707af25d359f, disabling compactions & flushes 2024-12-02T22:15:12,044 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,044 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,044 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. after waiting 0 ms 2024-12-02T22:15:12,044 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,047 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T22:15:12,047 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,047 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 87f06add23a346902905707af25d359f: Waiting for close lock at 1733177712043Running coprocessor pre-close hooks at 1733177712043Disabling compacts and flushes for region at 1733177712043Disabling writes for close at 1733177712044 (+1 ms)Writing region close event to WAL at 1733177712044Running coprocessor post-close hooks at 1733177712047 (+3 ms)Closed at 1733177712047 2024-12-02T22:15:12,047 WARN [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 87f06add23a346902905707af25d359f to self. 2024-12-02T22:15:12,049 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,049 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=CLOSED 2024-12-02T22:15:12,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:12,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-02T22:15:12,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 in 160 msec 2024-12-02T22:15:12,053 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE; state=CLOSED, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=true 2024-12-02T22:15:12,204 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE because future has completed 2024-12-02T22:15:12,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:12,359 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,359 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:15:12,359 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,359 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:15:12,360 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,360 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,361 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,361 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:12,361 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName A 2024-12-02T22:15:12,362 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:12,363 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:12,363 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,363 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:12,363 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName B 2024-12-02T22:15:12,364 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:12,364 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:12,364 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,364 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:12,364 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87f06add23a346902905707af25d359f columnFamilyName C 2024-12-02T22:15:12,365 DEBUG [StoreOpener-87f06add23a346902905707af25d359f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:12,365 INFO [StoreOpener-87f06add23a346902905707af25d359f-1 {}] regionserver.HStore(327): Store=87f06add23a346902905707af25d359f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:12,365 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,365 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,366 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:12,367 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:12,367 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,367 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,368 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:15:12,369 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,370 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 87f06add23a346902905707af25d359f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68983247, jitterRate=0.02793048322200775}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:15:12,370 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,370 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 87f06add23a346902905707af25d359f: Running coprocessor pre-open hook at 1733177712360Writing region info on filesystem at 1733177712360Initializing all the Stores at 1733177712360Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177712360Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177712360Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177712360Cleaning up temporary data from old regions at 1733177712367 (+7 ms)Running coprocessor post-open hooks at 1733177712370 (+3 ms)Region opened successfully at 1733177712370 2024-12-02T22:15:12,371 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., pid=98, masterSystemTime=1733177712357 2024-12-02T22:15:12,372 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,372 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=OPEN, openSeqNum=5, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:12,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-02T22:15:12,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 in 169 msec 2024-12-02T22:15:12,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T22:15:12,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, REOPEN/MOVE in 488 msec 2024-12-02T22:15:12,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-02T22:15:12,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-12-02T22:15:12,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-12-02T22:15:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-02T22:15:12,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:12,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f00c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429772a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,386 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6fb095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,388 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4b2cf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451c6d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d102257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298be083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc10a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,395 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a2a99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a73f500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434b7da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559bc9cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,400 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa6ca21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591a9d5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216237f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7a47ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a285181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3524d495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:12,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:12,409 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb61cdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:12,410 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367a53e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:12,411 DEBUG [Thread-1674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,411 DEBUG [Thread-1663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,411 DEBUG [Thread-1666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,411 DEBUG [Thread-1668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,412 DEBUG [Thread-1655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,412 DEBUG [Thread-1670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,412 DEBUG [Thread-1663 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,412 DEBUG [Thread-1672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,412 DEBUG [Thread-1663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,412 DEBUG [Thread-1661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,413 DEBUG [Thread-1657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,414 DEBUG [Thread-1659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:12,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:12,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-02T22:15:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:12,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:12,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:15:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:12,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:12,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:12,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029296d2eee0e244ff88abfc1f797c3075_87f06add23a346902905707af25d359f is 50, key is test_row_1/A:col10/1733177712443/Put/seqid=0 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177712478, max timestamp=1733177712492Num uniques 859; 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177712478, max timestamp=1733177712492Num uniques 859; 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177712478, max timestamp=1733177712492Num uniques 859; 2024-12-02T22:15:12,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:12,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742193_1369 (size=9714) 2024-12-02T22:15:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177772515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177772517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177772520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177772521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177772529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:12,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:12,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177772633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177772633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177772633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177772643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177772653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177772843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177772843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177772843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177772853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177772863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:12,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:12,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:12,902 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:12,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029296d2eee0e244ff88abfc1f797c3075_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029296d2eee0e244ff88abfc1f797c3075_87f06add23a346902905707af25d359f 2024-12-02T22:15:12,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/55589633df054925ad7cae4a3897e206, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:12,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/55589633df054925ad7cae4a3897e206 is 175, key is test_row_1/A:col10/1733177712443/Put/seqid=0 2024-12-02T22:15:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742194_1370 (size=22361) 2024-12-02T22:15:13,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:13,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177773163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177773163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177773163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177773173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177773173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,312 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/55589633df054925ad7cae4a3897e206 2024-12-02T22:15:13,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/05432990d6a54ba3a589af14c2f0b47e is 50, key is test_row_1/B:col10/1733177712443/Put/seqid=0 2024-12-02T22:15:13,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742195_1371 (size=9657) 2024-12-02T22:15:13,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:13,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177773673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177773673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177773683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177773683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177773693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:13,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/05432990d6a54ba3a589af14c2f0b47e 2024-12-02T22:15:13,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/00838939ba1f4f45aeca6febfce79190 is 50, key is test_row_1/C:col10/1733177712443/Put/seqid=0 2024-12-02T22:15:13,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742196_1372 (size=9657) 2024-12-02T22:15:13,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/00838939ba1f4f45aeca6febfce79190 2024-12-02T22:15:13,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/55589633df054925ad7cae4a3897e206 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206 2024-12-02T22:15:13,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206, entries=100, sequenceid=15, filesize=21.8 K 2024-12-02T22:15:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/05432990d6a54ba3a589af14c2f0b47e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e 2024-12-02T22:15:13,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e, entries=100, sequenceid=15, filesize=9.4 K 2024-12-02T22:15:13,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/00838939ba1f4f45aeca6febfce79190 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190 2024-12-02T22:15:13,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190, entries=100, sequenceid=15, filesize=9.4 K 2024-12-02T22:15:13,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 87f06add23a346902905707af25d359f in 1347ms, sequenceid=15, compaction requested=false 2024-12-02T22:15:13,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:13,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:13,802 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:13,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f7712edb0764cfa813714cafc9b8329_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177712526/Put/seqid=0 2024-12-02T22:15:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742197_1373 (size=12154) 2024-12-02T22:15:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f7712edb0764cfa813714cafc9b8329_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f7712edb0764cfa813714cafc9b8329_87f06add23a346902905707af25d359f 2024-12-02T22:15:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/36d2a6bb4f3d40e48df07be7b4afd842, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/36d2a6bb4f3d40e48df07be7b4afd842 is 175, key is test_row_0/A:col10/1733177712526/Put/seqid=0 2024-12-02T22:15:14,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742198_1374 (size=30955) 2024-12-02T22:15:14,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/36d2a6bb4f3d40e48df07be7b4afd842 2024-12-02T22:15:14,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/f777f25cfaa643cc89f0bd83d13d1205 is 50, key is test_row_0/B:col10/1733177712526/Put/seqid=0 2024-12-02T22:15:14,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742199_1375 (size=12001) 2024-12-02T22:15:14,322 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:15:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:14,660 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/f777f25cfaa643cc89f0bd83d13d1205 2024-12-02T22:15:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63ece42bf6e41b78248d3c4dd48e10a is 50, key is test_row_0/C:col10/1733177712526/Put/seqid=0 2024-12-02T22:15:14,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742200_1376 (size=12001) 2024-12-02T22:15:14,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63ece42bf6e41b78248d3c4dd48e10a 2024-12-02T22:15:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:14,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/36d2a6bb4f3d40e48df07be7b4afd842 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842 2024-12-02T22:15:14,703 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T22:15:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/f777f25cfaa643cc89f0bd83d13d1205 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205 2024-12-02T22:15:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T22:15:14,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63ece42bf6e41b78248d3c4dd48e10a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a 2024-12-02T22:15:14,712 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T22:15:14,713 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=20.13 KB/20610 for 87f06add23a346902905707af25d359f in 911ms, sequenceid=49, compaction requested=false 2024-12-02T22:15:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-02T22:15:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-02T22:15:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-02T22:15:14,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2940 sec 2024-12-02T22:15:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3040 sec 2024-12-02T22:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:14,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:15:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:14,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120298645298a30947a48d6793da7a46a2cb_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:14,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742201_1377 (size=19474) 2024-12-02T22:15:14,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733177714766, max timestamp=1733177714818Num uniques 859; 2024-12-02T22:15:14,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T22:15:14,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733177714766, max timestamp=1733177714818Num uniques 859; 2024-12-02T22:15:14,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T22:15:14,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733177714766, max timestamp=1733177714818Num uniques 859; 2024-12-02T22:15:14,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T22:15:14,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177774847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177774852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177774854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177774869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177774870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177774973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177774983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177774984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177774993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177774993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:15,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120298645298a30947a48d6793da7a46a2cb_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120298645298a30947a48d6793da7a46a2cb_87f06add23a346902905707af25d359f 2024-12-02T22:15:15,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/182772f78028417a950f1f4b55d11d87, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:15,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/182772f78028417a950f1f4b55d11d87 is 175, key is test_row_0/A:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:15,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742202_1378 (size=56733) 2024-12-02T22:15:15,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177775193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177775203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177775203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177775203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177775203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:15:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177775514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177775523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177775523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177775523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177775523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:15,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/182772f78028417a950f1f4b55d11d87 2024-12-02T22:15:15,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/374bd41d485a46e99ca9c93f51a295fa is 50, key is test_row_0/B:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:15,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742203_1379 (size=12001) 2024-12-02T22:15:15,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/374bd41d485a46e99ca9c93f51a295fa 2024-12-02T22:15:15,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/cc0910299e9443449dd136a79d2bd0b0 is 50, key is test_row_0/C:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:15,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742204_1380 (size=12001) 2024-12-02T22:15:15,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/cc0910299e9443449dd136a79d2bd0b0 2024-12-02T22:15:15,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/182772f78028417a950f1f4b55d11d87 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87 2024-12-02T22:15:15,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87, entries=300, sequenceid=59, filesize=55.4 K 2024-12-02T22:15:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/374bd41d485a46e99ca9c93f51a295fa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa 2024-12-02T22:15:15,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T22:15:15,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/cc0910299e9443449dd136a79d2bd0b0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0 2024-12-02T22:15:15,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T22:15:15,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 87f06add23a346902905707af25d359f in 913ms, sequenceid=59, compaction requested=true 2024-12-02T22:15:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:15,652 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:15,652 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:15,653 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:15,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:15,653 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:15,653 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:15,653 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:15,653 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:15,653 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=32.9 K 2024-12-02T22:15:15,653 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=107.5 K 2024-12-02T22:15:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:15,654 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:15,654 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87] 2024-12-02T22:15:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:15,654 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55589633df054925ad7cae4a3897e206, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177712441 2024-12-02T22:15:15,654 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 05432990d6a54ba3a589af14c2f0b47e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177712441 2024-12-02T22:15:15,654 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d2a6bb4f3d40e48df07be7b4afd842, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733177712498 2024-12-02T22:15:15,654 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f777f25cfaa643cc89f0bd83d13d1205, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733177712498 2024-12-02T22:15:15,655 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 182772f78028417a950f1f4b55d11d87, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714700 2024-12-02T22:15:15,655 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 374bd41d485a46e99ca9c93f51a295fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714735 2024-12-02T22:15:15,666 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:15,669 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:15,669 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/4986d898e4ef47fba8959fb73c47a927 is 50, key is test_row_0/B:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:15,671 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202662ae8ece0684e3fb595ac03bd3db19e_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:15,674 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202662ae8ece0684e3fb595ac03bd3db19e_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:15,674 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202662ae8ece0684e3fb595ac03bd3db19e_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742205_1381 (size=12104) 2024-12-02T22:15:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742206_1382 (size=4469) 2024-12-02T22:15:15,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/4986d898e4ef47fba8959fb73c47a927 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4986d898e4ef47fba8959fb73c47a927 2024-12-02T22:15:15,699 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into 4986d898e4ef47fba8959fb73c47a927(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:15,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:15,699 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=13, startTime=1733177715652; duration=0sec 2024-12-02T22:15:15,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:15,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:15,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:15,700 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:15,700 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:15,700 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:15,700 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=32.9 K 2024-12-02T22:15:15,701 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 00838939ba1f4f45aeca6febfce79190, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177712441 2024-12-02T22:15:15,701 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b63ece42bf6e41b78248d3c4dd48e10a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733177712498 2024-12-02T22:15:15,701 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0910299e9443449dd136a79d2bd0b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714735 2024-12-02T22:15:15,712 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:15,713 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a8aa9b1fc8b347e49ce1d23aaf1340cd is 50, key is test_row_0/C:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742207_1383 (size=12104) 2024-12-02T22:15:15,735 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a8aa9b1fc8b347e49ce1d23aaf1340cd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a8aa9b1fc8b347e49ce1d23aaf1340cd 2024-12-02T22:15:15,741 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into a8aa9b1fc8b347e49ce1d23aaf1340cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:15,741 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:15,741 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=13, startTime=1733177715654; duration=0sec 2024-12-02T22:15:15,741 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:15,741 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:16,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:16,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120234caa637c093459e8abbe8ba62b9fac1_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177714850/Put/seqid=0 2024-12-02T22:15:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742208_1384 (size=12154) 2024-12-02T22:15:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177776048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177776050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177776056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177776051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177776068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,089 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:16,089 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f0aa34144c5d4c4ab77b1525c77c1b85 is 175, key is test_row_0/A:col10/1733177714736/Put/seqid=0 2024-12-02T22:15:16,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742209_1385 (size=31058) 2024-12-02T22:15:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177776173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177776173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177776173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177776173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177776193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177776393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177776393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177776393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177776393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177776414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:16,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120234caa637c093459e8abbe8ba62b9fac1_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120234caa637c093459e8abbe8ba62b9fac1_87f06add23a346902905707af25d359f 2024-12-02T22:15:16,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e65691adc49f49ffa0d726023f137041, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:16,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e65691adc49f49ffa0d726023f137041 is 175, key is test_row_0/A:col10/1733177714850/Put/seqid=0 2024-12-02T22:15:16,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742210_1386 (size=30955) 2024-12-02T22:15:16,502 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f0aa34144c5d4c4ab77b1525c77c1b85 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85 2024-12-02T22:15:16,510 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into f0aa34144c5d4c4ab77b1525c77c1b85(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:16,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:16,510 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=13, startTime=1733177715652; duration=0sec 2024-12-02T22:15:16,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:16,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T22:15:16,583 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:16,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-02T22:15:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:16,586 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:16,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:16,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177776713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177776713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177776713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177776713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177776733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:16,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T22:15:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:16,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e65691adc49f49ffa0d726023f137041 2024-12-02T22:15:16,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T22:15:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:16,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:16,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/81776c654d514a98a4da4370e6245cfb is 50, key is test_row_0/B:col10/1733177714850/Put/seqid=0 2024-12-02T22:15:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:16,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742211_1387 (size=12001) 2024-12-02T22:15:16,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/81776c654d514a98a4da4370e6245cfb 2024-12-02T22:15:16,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/7d5bc46d3f7a4e47852b74c48943ca1d is 50, key is test_row_0/C:col10/1733177714850/Put/seqid=0 2024-12-02T22:15:16,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742212_1388 (size=12001) 2024-12-02T22:15:16,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/7d5bc46d3f7a4e47852b74c48943ca1d 2024-12-02T22:15:16,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e65691adc49f49ffa0d726023f137041 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041 2024-12-02T22:15:16,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041, entries=150, sequenceid=96, filesize=30.2 K 2024-12-02T22:15:16,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/81776c654d514a98a4da4370e6245cfb as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb 2024-12-02T22:15:16,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T22:15:16,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/7d5bc46d3f7a4e47852b74c48943ca1d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d 2024-12-02T22:15:16,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T22:15:16,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 87f06add23a346902905707af25d359f in 944ms, sequenceid=96, compaction requested=false 2024-12-02T22:15:16,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:17,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:17,051 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:17,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c5969c2433684a42ab55e9299a342b05_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177716049/Put/seqid=0 2024-12-02T22:15:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742213_1389 (size=12154) 2024-12-02T22:15:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:17,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177717277, max timestamp=1733177717311Num uniques 859; 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177717277, max timestamp=1733177717311Num uniques 859; 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177717277, max timestamp=1733177717311Num uniques 859; 2024-12-02T22:15:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:15:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177777353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177777355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177777357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177777359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177777374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177777483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:17,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c5969c2433684a42ab55e9299a342b05_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c5969c2433684a42ab55e9299a342b05_87f06add23a346902905707af25d359f 2024-12-02T22:15:17,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7aacf4ee3f8f435b935f598002765fdf, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:17,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7aacf4ee3f8f435b935f598002765fdf is 175, key is test_row_0/A:col10/1733177716049/Put/seqid=0 2024-12-02T22:15:17,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742214_1390 (size=30955) 2024-12-02T22:15:17,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177777503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177777703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177777703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177777703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177777703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177777723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:17,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7aacf4ee3f8f435b935f598002765fdf 2024-12-02T22:15:17,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/497da1d9ce154e93b425ca6ac1e03dac is 50, key is test_row_0/B:col10/1733177716049/Put/seqid=0 2024-12-02T22:15:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742215_1391 (size=12001) 2024-12-02T22:15:17,922 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/497da1d9ce154e93b425ca6ac1e03dac 2024-12-02T22:15:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b4c2fd1ee78741b194cbcb9b1f18ea76 is 50, key is test_row_0/C:col10/1733177716049/Put/seqid=0 2024-12-02T22:15:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742216_1392 (size=12001) 2024-12-02T22:15:17,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b4c2fd1ee78741b194cbcb9b1f18ea76 2024-12-02T22:15:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7aacf4ee3f8f435b935f598002765fdf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf 2024-12-02T22:15:17,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf, entries=150, sequenceid=106, filesize=30.2 K 2024-12-02T22:15:17,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/497da1d9ce154e93b425ca6ac1e03dac as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac 2024-12-02T22:15:17,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac, entries=150, sequenceid=106, filesize=11.7 K 2024-12-02T22:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b4c2fd1ee78741b194cbcb9b1f18ea76 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76 2024-12-02T22:15:17,973 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76, entries=150, sequenceid=106, filesize=11.7 K 2024-12-02T22:15:17,974 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 87f06add23a346902905707af25d359f in 922ms, sequenceid=106, compaction requested=true 2024-12-02T22:15:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-02T22:15:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-02T22:15:17,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T22:15:17,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2024-12-02T22:15:17,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.3990 sec 2024-12-02T22:15:18,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177717332, max timestamp=1733177718035Num uniques 1068; 2024-12-02T22:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177717332, max timestamp=1733177718035Num uniques 1068; 2024-12-02T22:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733177717332, max timestamp=1733177718035Num uniques 1068; 2024-12-02T22:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T22:15:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:18,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-02T22:15:18,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:18,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:18,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021240ede969a04f2eb4d4dbb6b0fa7096_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742217_1393 (size=14794) 2024-12-02T22:15:18,079 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:18,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021240ede969a04f2eb4d4dbb6b0fa7096_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021240ede969a04f2eb4d4dbb6b0fa7096_87f06add23a346902905707af25d359f 2024-12-02T22:15:18,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f2c38aab237f4fbc9279b3ffb19892fb, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:18,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f2c38aab237f4fbc9279b3ffb19892fb is 175, key is test_row_0/A:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742218_1394 (size=39749) 2024-12-02T22:15:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177718078, max timestamp=1733177718101Num uniques 859; 2024-12-02T22:15:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177718078, max timestamp=1733177718101Num uniques 859; 2024-12-02T22:15:18,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177718078, max timestamp=1733177718101Num uniques 859; 2024-12-02T22:15:18,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177778132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177778135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177778137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177778148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177778148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177778253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177778263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177778263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177778263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177778263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177778473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177778483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177778483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f2c38aab237f4fbc9279b3ffb19892fb 2024-12-02T22:15:18,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/cbfc7622e53e4c48bd81379ebd0ad524 is 50, key is test_row_0/B:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742219_1395 (size=12151) 2024-12-02T22:15:18,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/cbfc7622e53e4c48bd81379ebd0ad524 2024-12-02T22:15:18,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177778493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177778493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/ac82f277edd744688d17b50362bf97b0 is 50, key is test_row_0/C:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742220_1396 (size=12151) 2024-12-02T22:15:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T22:15:18,733 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:18,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-02T22:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:18,735 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:18,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:18,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177778793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177778793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177778794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177778813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177778813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:18,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T22:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:18,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:18,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/ac82f277edd744688d17b50362bf97b0 2024-12-02T22:15:18,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/f2c38aab237f4fbc9279b3ffb19892fb as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb 2024-12-02T22:15:18,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb, entries=200, sequenceid=148, filesize=38.8 K 2024-12-02T22:15:18,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/cbfc7622e53e4c48bd81379ebd0ad524 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524 2024-12-02T22:15:18,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524, entries=150, sequenceid=148, filesize=11.9 K 2024-12-02T22:15:18,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/ac82f277edd744688d17b50362bf97b0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0 2024-12-02T22:15:18,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0, entries=150, sequenceid=148, filesize=11.9 K 2024-12-02T22:15:18,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for 87f06add23a346902905707af25d359f in 876ms, sequenceid=148, compaction requested=true 2024-12-02T22:15:18,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:18,930 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:18,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:15:18,930 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:18,931 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,931 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,931 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=129.6 K 2024-12-02T22:15:18,931 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4986d898e4ef47fba8959fb73c47a927, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=47.1 K 2024-12-02T22:15:18,931 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb] 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4986d898e4ef47fba8959fb73c47a927, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714735 2024-12-02T22:15:18,931 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0aa34144c5d4c4ab77b1525c77c1b85, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714735 2024-12-02T22:15:18,932 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 81776c654d514a98a4da4370e6245cfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733177714850 2024-12-02T22:15:18,932 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65691adc49f49ffa0d726023f137041, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733177714850 2024-12-02T22:15:18,932 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 497da1d9ce154e93b425ca6ac1e03dac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733177716045 2024-12-02T22:15:18,932 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aacf4ee3f8f435b935f598002765fdf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733177716045 2024-12-02T22:15:18,932 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting cbfc7622e53e4c48bd81379ebd0ad524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:18,933 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c38aab237f4fbc9279b3ffb19892fb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:18,941 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:18,945 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412024707ccd119374a9599e9f90448538722_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:18,947 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:18,948 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/f5fff81263c14357bee21db3f05c20c8 is 50, key is test_row_0/B:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,948 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412024707ccd119374a9599e9f90448538722_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:18,948 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024707ccd119374a9599e9f90448538722_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742221_1397 (size=12391) 2024-12-02T22:15:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742222_1398 (size=4469) 2024-12-02T22:15:18,965 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#339 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:18,966 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/6783ef4f6f844c3ba5b2ebc31fd69d9d is 175, key is test_row_0/A:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:18,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742223_1399 (size=31345) 2024-12-02T22:15:18,979 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/6783ef4f6f844c3ba5b2ebc31fd69d9d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d 2024-12-02T22:15:18,985 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into 6783ef4f6f844c3ba5b2ebc31fd69d9d(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:18,985 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:18,985 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=12, startTime=1733177718929; duration=0sec 2024-12-02T22:15:18,985 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:18,985 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:18,985 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:18,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:18,987 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:18,987 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:18,987 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a8aa9b1fc8b347e49ce1d23aaf1340cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=47.1 K 2024-12-02T22:15:18,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8aa9b1fc8b347e49ce1d23aaf1340cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177714735 2024-12-02T22:15:18,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5bc46d3f7a4e47852b74c48943ca1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733177714850 2024-12-02T22:15:18,989 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c2fd1ee78741b194cbcb9b1f18ea76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733177716045 2024-12-02T22:15:18,990 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac82f277edd744688d17b50362bf97b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:19,002 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:19,002 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/35b5a063dd25426789eb97db0dc26437 is 50, key is test_row_0/C:col10/1733177718042/Put/seqid=0 2024-12-02T22:15:19,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742224_1400 (size=12391) 2024-12-02T22:15:19,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T22:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:19,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T22:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:19,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:19,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020976682243dc4d368c36f4671308bc1a_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177718134/Put/seqid=0 2024-12-02T22:15:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:19,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742225_1401 (size=12304) 2024-12-02T22:15:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177779333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177779338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177779339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177779340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177779346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,365 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/f5fff81263c14357bee21db3f05c20c8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f5fff81263c14357bee21db3f05c20c8 2024-12-02T22:15:19,369 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into f5fff81263c14357bee21db3f05c20c8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:19,369 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:19,369 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=12, startTime=1733177718930; duration=0sec 2024-12-02T22:15:19,370 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:19,370 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:19,417 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/35b5a063dd25426789eb97db0dc26437 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/35b5a063dd25426789eb97db0dc26437 2024-12-02T22:15:19,422 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into 35b5a063dd25426789eb97db0dc26437(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:19,422 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:19,422 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=12, startTime=1733177718930; duration=0sec 2024-12-02T22:15:19,422 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:19,422 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177779453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,465 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020976682243dc4d368c36f4671308bc1a_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020976682243dc4d368c36f4671308bc1a_87f06add23a346902905707af25d359f 2024-12-02T22:15:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/b215f38761a347129a47e2fbe1d03837, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:19,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/b215f38761a347129a47e2fbe1d03837 is 175, key is test_row_0/A:col10/1733177718134/Put/seqid=0 2024-12-02T22:15:19,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742226_1402 (size=31105) 2024-12-02T22:15:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177779464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177779464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177779473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177779673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177779683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177779684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177779684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:19,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/b215f38761a347129a47e2fbe1d03837 2024-12-02T22:15:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:19,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/17ff5effe6674b708332e90420e49a19 is 50, key is test_row_0/B:col10/1733177718134/Put/seqid=0 2024-12-02T22:15:19,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742227_1403 (size=12151) 2024-12-02T22:15:19,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/17ff5effe6674b708332e90420e49a19 2024-12-02T22:15:19,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a30888f029f04ba7b589f49a5555bc2a is 50, key is test_row_0/C:col10/1733177718134/Put/seqid=0 2024-12-02T22:15:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742228_1404 (size=12151) 2024-12-02T22:15:19,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a30888f029f04ba7b589f49a5555bc2a 2024-12-02T22:15:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/b215f38761a347129a47e2fbe1d03837 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837 2024-12-02T22:15:19,925 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837, entries=150, sequenceid=184, filesize=30.4 K 2024-12-02T22:15:19,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/17ff5effe6674b708332e90420e49a19 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19 2024-12-02T22:15:19,930 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19, entries=150, sequenceid=184, filesize=11.9 K 2024-12-02T22:15:19,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a30888f029f04ba7b589f49a5555bc2a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a 2024-12-02T22:15:19,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-02T22:15:19,936 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 87f06add23a346902905707af25d359f in 895ms, sequenceid=184, compaction requested=false 2024-12-02T22:15:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-02T22:15:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-02T22:15:19,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-02T22:15:19,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-12-02T22:15:19,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2070 sec 2024-12-02T22:15:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:19,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:20,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fd1127f96b0f40aab381eb9a31c9a138_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742229_1405 (size=14794) 2024-12-02T22:15:20,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:20,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fd1127f96b0f40aab381eb9a31c9a138_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd1127f96b0f40aab381eb9a31c9a138_87f06add23a346902905707af25d359f 2024-12-02T22:15:20,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/1e5a025d3fa546dabeed522f6be8a8c3, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/1e5a025d3fa546dabeed522f6be8a8c3 is 175, key is test_row_0/A:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742230_1406 (size=39749) 2024-12-02T22:15:20,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/1e5a025d3fa546dabeed522f6be8a8c3 2024-12-02T22:15:20,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/82e195aae238449f9f4022c5e5afb619 is 50, key is test_row_0/B:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742231_1407 (size=12151) 2024-12-02T22:15:20,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/82e195aae238449f9f4022c5e5afb619 2024-12-02T22:15:20,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d30837dce25549dd9cf6ad6d76f96622 is 50, key is test_row_0/C:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177780054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177720043, max timestamp=1733177720066Num uniques 859; 2024-12-02T22:15:20,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177720043, max timestamp=1733177720066Num uniques 859; 2024-12-02T22:15:20,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:20,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177720043, max timestamp=1733177720066Num uniques 859; 2024-12-02T22:15:20,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:15:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742232_1408 (size=12151) 2024-12-02T22:15:20,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d30837dce25549dd9cf6ad6d76f96622 2024-12-02T22:15:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/1e5a025d3fa546dabeed522f6be8a8c3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3 2024-12-02T22:15:20,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3, entries=200, sequenceid=197, filesize=38.8 K 2024-12-02T22:15:20,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/82e195aae238449f9f4022c5e5afb619 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619 2024-12-02T22:15:20,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619, entries=150, sequenceid=197, filesize=11.9 K 2024-12-02T22:15:20,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d30837dce25549dd9cf6ad6d76f96622 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622 2024-12-02T22:15:20,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622, entries=150, sequenceid=197, filesize=11.9 K 2024-12-02T22:15:20,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for 87f06add23a346902905707af25d359f in 105ms, sequenceid=197, compaction requested=true 2024-12-02T22:15:20,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:20,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:20,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:20,102 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:20,102 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:20,103 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:20,103 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:20,103 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:20,103 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:20,103 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:20,103 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:20,103 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=99.8 K 2024-12-02T22:15:20,104 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f5fff81263c14357bee21db3f05c20c8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=35.8 K 2024-12-02T22:15:20,104 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:20,104 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3] 2024-12-02T22:15:20,104 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fff81263c14357bee21db3f05c20c8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:20,104 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6783ef4f6f844c3ba5b2ebc31fd69d9d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:20,104 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ff5effe6674b708332e90420e49a19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177718134 2024-12-02T22:15:20,105 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b215f38761a347129a47e2fbe1d03837, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177718134 2024-12-02T22:15:20,105 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e195aae238449f9f4022c5e5afb619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:20,105 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5a025d3fa546dabeed522f6be8a8c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:20,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-12-02T22:15:20,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:20,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:20,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:20,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:20,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:20,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:20,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027bb9f9fc69634e4887fb75fceac7c83d_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177720099/Put/seqid=0 2024-12-02T22:15:20,133 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,139 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:20,139 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/0cfdb5d9464449529015c851e98e409c is 50, key is test_row_0/B:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,140 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202cf2b096a67024afc8219ac326d8ffe35_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,142 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202cf2b096a67024afc8219ac326d8ffe35_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,142 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cf2b096a67024afc8219ac326d8ffe35_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742234_1410 (size=12493) 2024-12-02T22:15:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742235_1411 (size=4469) 2024-12-02T22:15:20,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742233_1409 (size=14794) 2024-12-02T22:15:20,163 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#349 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:20,164 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a0c5b63dc41e417cb5c98a528e138324 is 175, key is test_row_0/A:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742236_1412 (size=31447) 2024-12-02T22:15:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177780173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177780182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177780185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177780186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177780293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177780313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177780313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177780363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,381 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 87f06add23a346902905707af25d359f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177780393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177780513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177780523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177780523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,559 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/0cfdb5d9464449529015c851e98e409c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/0cfdb5d9464449529015c851e98e409c 2024-12-02T22:15:20,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:20,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027bb9f9fc69634e4887fb75fceac7c83d_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027bb9f9fc69634e4887fb75fceac7c83d_87f06add23a346902905707af25d359f 2024-12-02T22:15:20,564 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into 0cfdb5d9464449529015c851e98e409c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:20,564 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:20,564 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=13, startTime=1733177720102; duration=0sec 2024-12-02T22:15:20,564 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:20,564 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:20,565 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:20,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/98a59c8574ad43c584cec60fb0ac0671, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:20,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/98a59c8574ad43c584cec60fb0ac0671 is 175, key is test_row_0/A:col10/1733177720099/Put/seqid=0 2024-12-02T22:15:20,567 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:20,568 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:20,568 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:20,568 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/35b5a063dd25426789eb97db0dc26437, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=35.8 K 2024-12-02T22:15:20,568 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b5a063dd25426789eb97db0dc26437, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733177717369 2024-12-02T22:15:20,568 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a30888f029f04ba7b589f49a5555bc2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733177718134 2024-12-02T22:15:20,569 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d30837dce25549dd9cf6ad6d76f96622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:20,573 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a0c5b63dc41e417cb5c98a528e138324 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324 2024-12-02T22:15:20,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742237_1413 (size=39749) 2024-12-02T22:15:20,575 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/98a59c8574ad43c584cec60fb0ac0671 2024-12-02T22:15:20,582 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into a0c5b63dc41e417cb5c98a528e138324(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:20,582 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:20,582 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=13, startTime=1733177720102; duration=0sec 2024-12-02T22:15:20,582 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:20,582 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:20,590 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:20,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/33f2a960a6384d789597142d6771edba is 50, key is test_row_0/B:col10/1733177720099/Put/seqid=0 2024-12-02T22:15:20,591 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/c3404874e4574258a48f1645b6501cc1 is 50, key is test_row_0/C:col10/1733177719996/Put/seqid=0 2024-12-02T22:15:20,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742238_1414 (size=12151) 2024-12-02T22:15:20,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/33f2a960a6384d789597142d6771edba 2024-12-02T22:15:20,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742239_1415 (size=12493) 2024-12-02T22:15:20,607 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/c3404874e4574258a48f1645b6501cc1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c3404874e4574258a48f1645b6501cc1 2024-12-02T22:15:20,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a361d98dab3b403983d8c5da9ab80e7e is 50, key is test_row_0/C:col10/1733177720099/Put/seqid=0 2024-12-02T22:15:20,611 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into c3404874e4574258a48f1645b6501cc1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:20,611 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:20,611 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=13, startTime=1733177720103; duration=0sec 2024-12-02T22:15:20,611 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:20,611 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:20,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742240_1416 (size=12151) 2024-12-02T22:15:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177780713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177780843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177780843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177780843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T22:15:20,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:20,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T22:15:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:20,885 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:20,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:20,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:21,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a361d98dab3b403983d8c5da9ab80e7e 2024-12-02T22:15:21,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/98a59c8574ad43c584cec60fb0ac0671 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671 2024-12-02T22:15:21,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671, entries=200, sequenceid=225, filesize=38.8 K 2024-12-02T22:15:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/33f2a960a6384d789597142d6771edba as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba 2024-12-02T22:15:21,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba, entries=150, sequenceid=225, filesize=11.9 K 2024-12-02T22:15:21,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a361d98dab3b403983d8c5da9ab80e7e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e 2024-12-02T22:15:21,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T22:15:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:21,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:21,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e, entries=150, sequenceid=225, filesize=11.9 K 2024-12-02T22:15:21,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=114.05 KB/116790 for 87f06add23a346902905707af25d359f in 936ms, sequenceid=225, compaction requested=false 2024-12-02T22:15:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:21,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T22:15:21,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:21,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f984895680044399982d62d184ff8c1e_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177720183/Put/seqid=0 2024-12-02T22:15:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:21,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742241_1417 (size=12304) 2024-12-02T22:15:21,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:21,213 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f984895680044399982d62d184ff8c1e_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f984895680044399982d62d184ff8c1e_87f06add23a346902905707af25d359f 2024-12-02T22:15:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7a5e0b127f984a4c899c99c29bea30b4, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:21,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7a5e0b127f984a4c899c99c29bea30b4 is 175, key is test_row_0/A:col10/1733177720183/Put/seqid=0 2024-12-02T22:15:21,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742242_1418 (size=31105) 2024-12-02T22:15:21,220 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7a5e0b127f984a4c899c99c29bea30b4 2024-12-02T22:15:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:21,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/49af0f85ebce454eac16781aa1c49245 is 50, key is test_row_0/B:col10/1733177720183/Put/seqid=0 2024-12-02T22:15:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742243_1419 (size=12151) 2024-12-02T22:15:21,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177781282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177781363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177781363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177781363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177781393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:21,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177781613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:21,635 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/49af0f85ebce454eac16781aa1c49245 2024-12-02T22:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a8151ff87524ce6ae1e93c06435331c is 50, key is test_row_0/C:col10/1733177720183/Put/seqid=0 2024-12-02T22:15:21,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742244_1420 (size=12151) 2024-12-02T22:15:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177781933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:22,051 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a8151ff87524ce6ae1e93c06435331c 2024-12-02T22:15:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7a5e0b127f984a4c899c99c29bea30b4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4 2024-12-02T22:15:22,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4, entries=150, sequenceid=248, filesize=30.4 K 2024-12-02T22:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/49af0f85ebce454eac16781aa1c49245 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245 2024-12-02T22:15:22,068 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245, entries=150, sequenceid=248, filesize=11.9 K 2024-12-02T22:15:22,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a8151ff87524ce6ae1e93c06435331c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c 2024-12-02T22:15:22,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c, entries=150, sequenceid=248, filesize=11.9 K 2024-12-02T22:15:22,075 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 87f06add23a346902905707af25d359f in 883ms, sequenceid=248, compaction requested=true 2024-12-02T22:15:22,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:22,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:22,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T22:15:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T22:15:22,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T22:15:22,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1900 sec 2024-12-02T22:15:22,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1970 sec 2024-12-02T22:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:22,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:22,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028912be79a805482abab9b183ed40e0ba_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:22,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742245_1421 (size=14994) 2024-12-02T22:15:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177782421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177782423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177782435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177782436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177782453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177782543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177782543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177782553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177782553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177782753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177782763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177782773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177782773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:22,801 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:22,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028912be79a805482abab9b183ed40e0ba_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028912be79a805482abab9b183ed40e0ba_87f06add23a346902905707af25d359f 2024-12-02T22:15:22,805 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/67580b8230074fd8b71f55b22dbdfc9d, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:22,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/67580b8230074fd8b71f55b22dbdfc9d is 175, key is test_row_0/A:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:22,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742246_1422 (size=39949) 2024-12-02T22:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T22:15:23,033 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:23,034 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-02T22:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:23,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:23,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:23,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177783073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177783083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177783093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177783093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:23,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T22:15:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:23,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/67580b8230074fd8b71f55b22dbdfc9d 2024-12-02T22:15:23,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/9a561b987c5148a18c4a93d6486fe7e7 is 50, key is test_row_0/B:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:23,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742247_1423 (size=12301) 2024-12-02T22:15:23,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/9a561b987c5148a18c4a93d6486fe7e7 2024-12-02T22:15:23,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/0120c40896bd40cba929b2e6c34537a8 is 50, key is test_row_0/C:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:23,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742248_1424 (size=12301) 2024-12-02T22:15:23,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/0120c40896bd40cba929b2e6c34537a8 2024-12-02T22:15:23,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/67580b8230074fd8b71f55b22dbdfc9d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d 2024-12-02T22:15:23,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d, entries=200, sequenceid=261, filesize=39.0 K 2024-12-02T22:15:23,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/9a561b987c5148a18c4a93d6486fe7e7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7 2024-12-02T22:15:23,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7, entries=150, sequenceid=261, filesize=12.0 K 2024-12-02T22:15:23,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/0120c40896bd40cba929b2e6c34537a8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8 2024-12-02T22:15:23,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8, entries=150, sequenceid=261, filesize=12.0 K 2024-12-02T22:15:23,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 87f06add23a346902905707af25d359f in 870ms, sequenceid=261, compaction requested=true 2024-12-02T22:15:23,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:23,258 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:23,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:23,258 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:23,259 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:23,259 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:23,259 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,259 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=138.9 K 2024-12-02T22:15:23,259 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,260 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d] 2024-12-02T22:15:23,260 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:23,260 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:23,260 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,260 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/0cfdb5d9464449529015c851e98e409c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=47.9 K 2024-12-02T22:15:23,261 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c5b63dc41e417cb5c98a528e138324, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:23,261 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfdb5d9464449529015c851e98e409c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:23,261 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a59c8574ad43c584cec60fb0ac0671, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733177720078 2024-12-02T22:15:23,262 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f2a960a6384d789597142d6771edba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733177720078 2024-12-02T22:15:23,262 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5e0b127f984a4c899c99c29bea30b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733177720169 2024-12-02T22:15:23,264 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 49af0f85ebce454eac16781aa1c49245, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733177720169 2024-12-02T22:15:23,264 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67580b8230074fd8b71f55b22dbdfc9d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721265 2024-12-02T22:15:23,264 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a561b987c5148a18c4a93d6486fe7e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721275 2024-12-02T22:15:23,281 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:23,285 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:23,285 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/8a0a5aee1c054ffda538f10b0d726df6 is 50, key is test_row_0/B:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:23,289 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202503bc2a8499d414da40e2e89b22dd97e_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:23,292 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202503bc2a8499d414da40e2e89b22dd97e_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:23,292 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202503bc2a8499d414da40e2e89b22dd97e_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:23,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742249_1425 (size=12779) 2024-12-02T22:15:23,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742250_1426 (size=4469) 2024-12-02T22:15:23,312 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#360 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:23,312 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/ea31337ca5434124b517cf99aa77bd74 is 175, key is test_row_0/A:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:23,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742251_1427 (size=31733) 2024-12-02T22:15:23,326 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/ea31337ca5434124b517cf99aa77bd74 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74 2024-12-02T22:15:23,331 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into ea31337ca5434124b517cf99aa77bd74(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:23,331 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:23,331 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=12, startTime=1733177723257; duration=0sec 2024-12-02T22:15:23,331 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:23,331 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:23,331 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:23,332 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:23,332 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:23,332 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,332 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c3404874e4574258a48f1645b6501cc1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=47.9 K 2024-12-02T22:15:23,333 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3404874e4574258a48f1645b6501cc1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733177719336 2024-12-02T22:15:23,334 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a361d98dab3b403983d8c5da9ab80e7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733177720078 2024-12-02T22:15:23,334 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8151ff87524ce6ae1e93c06435331c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733177720169 2024-12-02T22:15:23,334 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0120c40896bd40cba929b2e6c34537a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721275 2024-12-02T22:15:23,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T22:15:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:23,342 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:23,345 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:23,345 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/4f496f035c314584ae99d546726afc91 is 50, key is test_row_0/C:col10/1733177722386/Put/seqid=0 2024-12-02T22:15:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:23,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fcb756ec5737434abbfcc5727ffd64c0_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177722435/Put/seqid=0 2024-12-02T22:15:23,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742252_1428 (size=12779) 2024-12-02T22:15:23,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742253_1429 (size=12454) 2024-12-02T22:15:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:23,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fcb756ec5737434abbfcc5727ffd64c0_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fcb756ec5737434abbfcc5727ffd64c0_87f06add23a346902905707af25d359f 2024-12-02T22:15:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/21d107dfcc5d4146bac194fc987ad4b3, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/21d107dfcc5d4146bac194fc987ad4b3 is 175, key is test_row_0/A:col10/1733177722435/Put/seqid=0 2024-12-02T22:15:23,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742254_1430 (size=31255) 2024-12-02T22:15:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:23,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:23,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177783557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177783583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177783603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177783613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177783614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177783683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:23,710 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/8a0a5aee1c054ffda538f10b0d726df6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8a0a5aee1c054ffda538f10b0d726df6 2024-12-02T22:15:23,714 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into 8a0a5aee1c054ffda538f10b0d726df6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:23,714 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:23,715 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=12, startTime=1733177723258; duration=0sec 2024-12-02T22:15:23,715 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:23,715 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:23,765 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/4f496f035c314584ae99d546726afc91 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/4f496f035c314584ae99d546726afc91 2024-12-02T22:15:23,769 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into 4f496f035c314584ae99d546726afc91(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:23,769 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:23,769 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=12, startTime=1733177723258; duration=0sec 2024-12-02T22:15:23,769 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:23,769 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:23,794 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/21d107dfcc5d4146bac194fc987ad4b3 2024-12-02T22:15:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/e7266e07a5634cdb9c2b53e04d3a773e is 50, key is test_row_0/B:col10/1733177722435/Put/seqid=0 2024-12-02T22:15:23,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742255_1431 (size=12301) 2024-12-02T22:15:23,806 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/e7266e07a5634cdb9c2b53e04d3a773e 2024-12-02T22:15:23,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/c744116e32504ad5bf5d2e43c28e6f22 is 50, key is test_row_0/C:col10/1733177722435/Put/seqid=0 2024-12-02T22:15:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742256_1432 (size=12301) 2024-12-02T22:15:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177783903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:24,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/c744116e32504ad5bf5d2e43c28e6f22 2024-12-02T22:15:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/21d107dfcc5d4146bac194fc987ad4b3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3 2024-12-02T22:15:24,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3, entries=150, sequenceid=281, filesize=30.5 K 2024-12-02T22:15:24,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/e7266e07a5634cdb9c2b53e04d3a773e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e 2024-12-02T22:15:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177784223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,231 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T22:15:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/c744116e32504ad5bf5d2e43c28e6f22 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22 2024-12-02T22:15:24,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T22:15:24,237 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87f06add23a346902905707af25d359f in 895ms, sequenceid=281, compaction requested=false 2024-12-02T22:15:24,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:24,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:24,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-02T22:15:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-02T22:15:24,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T22:15:24,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2020 sec 2024-12-02T22:15:24,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.2070 sec 2024-12-02T22:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:24,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:15:24,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:24,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b312617f637b4a728b2d19be4019e3f1_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:24,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742257_1433 (size=14994) 2024-12-02T22:15:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177784663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177784664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177784698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177784698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177784743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177784803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177784803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177784813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177784813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,028 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177785023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177785023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177785023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177785023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b312617f637b4a728b2d19be4019e3f1_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b312617f637b4a728b2d19be4019e3f1_87f06add23a346902905707af25d359f 2024-12-02T22:15:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/cc57f10d9574471b9d876a6320ffb878, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/cc57f10d9574471b9d876a6320ffb878 is 175, key is test_row_0/A:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742258_1434 (size=39949) 2024-12-02T22:15:25,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/cc57f10d9574471b9d876a6320ffb878 2024-12-02T22:15:25,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/dffb2b3d051b49bba77d61bbb8fdbabd is 50, key is test_row_0/B:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742259_1435 (size=12301) 2024-12-02T22:15:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T22:15:25,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:25,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-02T22:15:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:25,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:25,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:25,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:25,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T22:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177785343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177785344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177785344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177785344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/dffb2b3d051b49bba77d61bbb8fdbabd 2024-12-02T22:15:25,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5fae4725bf034601924cc756915ecba3 is 50, key is test_row_0/C:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742260_1436 (size=12301) 2024-12-02T22:15:25,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T22:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:25,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T22:15:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177785763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,766 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 87f06add23a346902905707af25d359f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T22:15:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177785853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177785853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5fae4725bf034601924cc756915ecba3 2024-12-02T22:15:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177785863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177785863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/cc57f10d9574471b9d876a6320ffb878 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878 2024-12-02T22:15:25,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878, entries=200, sequenceid=297, filesize=39.0 K 2024-12-02T22:15:25,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/dffb2b3d051b49bba77d61bbb8fdbabd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd 2024-12-02T22:15:25,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd, entries=150, sequenceid=297, filesize=12.0 K 2024-12-02T22:15:25,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5fae4725bf034601924cc756915ecba3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3 2024-12-02T22:15:25,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3, entries=150, sequenceid=297, filesize=12.0 K 2024-12-02T22:15:25,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 87f06add23a346902905707af25d359f in 1282ms, sequenceid=297, compaction requested=true 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:25,898 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:25,898 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:25,899 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:25,899 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:25,899 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,899 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=100.5 K 2024-12-02T22:15:25,899 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,900 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878] 2024-12-02T22:15:25,900 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea31337ca5434124b517cf99aa77bd74, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721275 2024-12-02T22:15:25,902 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:25,903 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:25,903 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,903 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8a0a5aee1c054ffda538f10b0d726df6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=36.5 K 2024-12-02T22:15:25,905 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d107dfcc5d4146bac194fc987ad4b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733177722419 2024-12-02T22:15:25,905 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0a5aee1c054ffda538f10b0d726df6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721275 2024-12-02T22:15:25,905 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc57f10d9574471b9d876a6320ffb878, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723540 2024-12-02T22:15:25,906 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e7266e07a5634cdb9c2b53e04d3a773e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733177722419 2024-12-02T22:15:25,907 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting dffb2b3d051b49bba77d61bbb8fdbabd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723549 2024-12-02T22:15:25,917 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,925 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d128d08b52e54ec78c256b2fe9dbfc92_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,927 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:25,927 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/fe59f3a95d4c4738b9bad4f22e0230f5 is 50, key is test_row_0/B:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,927 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d128d08b52e54ec78c256b2fe9dbfc92_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,928 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d128d08b52e54ec78c256b2fe9dbfc92_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742262_1438 (size=4469) 2024-12-02T22:15:25,940 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#369 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:25,941 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a1d8366bdd6e470ea1beee9a9a5fa243 is 175, key is test_row_0/A:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742261_1437 (size=12881) 2024-12-02T22:15:25,952 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/fe59f3a95d4c4738b9bad4f22e0230f5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fe59f3a95d4c4738b9bad4f22e0230f5 2024-12-02T22:15:25,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T22:15:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:25,961 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into fe59f3a95d4c4738b9bad4f22e0230f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:25,961 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:25,961 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=13, startTime=1733177725898; duration=0sec 2024-12-02T22:15:25,962 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:25,962 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:25,962 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:25,963 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:25,963 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:25,963 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:25,963 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/4f496f035c314584ae99d546726afc91, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=36.5 K 2024-12-02T22:15:25,963 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f496f035c314584ae99d546726afc91, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733177721275 2024-12-02T22:15:25,963 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c744116e32504ad5bf5d2e43c28e6f22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733177722419 2024-12-02T22:15:25,964 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fae4725bf034601924cc756915ecba3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723549 2024-12-02T22:15:25,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742263_1439 (size=31835) 2024-12-02T22:15:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b0264f5632b34d05803d40219886a1ce_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177724651/Put/seqid=0 2024-12-02T22:15:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742264_1440 (size=12454) 2024-12-02T22:15:25,974 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a1d8366bdd6e470ea1beee9a9a5fa243 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243 2024-12-02T22:15:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:25,979 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into a1d8366bdd6e470ea1beee9a9a5fa243(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:25,979 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:25,979 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=13, startTime=1733177725898; duration=0sec 2024-12-02T22:15:25,979 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:25,979 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:25,980 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b0264f5632b34d05803d40219886a1ce_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b0264f5632b34d05803d40219886a1ce_87f06add23a346902905707af25d359f 2024-12-02T22:15:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/71f872dd5dee4a4ab069a3d1f8e2304b, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/71f872dd5dee4a4ab069a3d1f8e2304b is 175, key is test_row_0/A:col10/1733177724651/Put/seqid=0 2024-12-02T22:15:25,986 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:25,986 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a60ccfd75fb546b894a0a4e458b9d0d8 is 50, key is test_row_0/C:col10/1733177723549/Put/seqid=0 2024-12-02T22:15:25,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742265_1441 (size=31255) 2024-12-02T22:15:26,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742266_1442 (size=12881) 2024-12-02T22:15:26,009 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/a60ccfd75fb546b894a0a4e458b9d0d8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a60ccfd75fb546b894a0a4e458b9d0d8 2024-12-02T22:15:26,014 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into a60ccfd75fb546b894a0a4e458b9d0d8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:26,014 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:26,014 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=13, startTime=1733177725898; duration=0sec 2024-12-02T22:15:26,014 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:26,015 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:26,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/71f872dd5dee4a4ab069a3d1f8e2304b 2024-12-02T22:15:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/75faa09078e94d6e89dcc75b3912da64 is 50, key is test_row_0/B:col10/1733177724651/Put/seqid=0 2024-12-02T22:15:26,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742267_1443 (size=12301) 2024-12-02T22:15:26,804 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/75faa09078e94d6e89dcc75b3912da64 2024-12-02T22:15:26,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f6f6e490d0904e4dafd4f11cf03c80f9 is 50, key is test_row_0/C:col10/1733177724651/Put/seqid=0 2024-12-02T22:15:26,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742268_1444 (size=12301) 2024-12-02T22:15:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:26,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177786897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177786897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177786898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177786899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177787013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177787014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177787014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177787014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,217 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f6f6e490d0904e4dafd4f11cf03c80f9 2024-12-02T22:15:27,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/71f872dd5dee4a4ab069a3d1f8e2304b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b 2024-12-02T22:15:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177787224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177787224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177787224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,234 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b, entries=150, sequenceid=316, filesize=30.5 K 2024-12-02T22:15:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/75faa09078e94d6e89dcc75b3912da64 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64 2024-12-02T22:15:27,238 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T22:15:27,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f6f6e490d0904e4dafd4f11cf03c80f9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9 2024-12-02T22:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177787233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,242 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T22:15:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87f06add23a346902905707af25d359f in 1287ms, sequenceid=316, compaction requested=false 2024-12-02T22:15:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-02T22:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-02T22:15:27,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T22:15:27,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-12-02T22:15:27,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.0640 sec 2024-12-02T22:15:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T22:15:27,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:27,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-02T22:15:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T22:15:27,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:27,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:27,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T22:15:27,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T22:15:27,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:27,488 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028560bb73e36c4edb8c69b1a1942f99ac_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177726898/Put/seqid=0 2024-12-02T22:15:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742269_1445 (size=12454) 2024-12-02T22:15:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:27,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:27,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177787581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177787582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177787590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177787591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T22:15:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177787693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177787703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177787703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177787703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177787783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,789 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 87f06add23a346902905707af25d359f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:27,904 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028560bb73e36c4edb8c69b1a1942f99ac_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028560bb73e36c4edb8c69b1a1942f99ac_87f06add23a346902905707af25d359f 2024-12-02T22:15:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/2a98711fbccd41a4984dadeaef57d723, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/2a98711fbccd41a4984dadeaef57d723 is 175, key is test_row_0/A:col10/1733177726898/Put/seqid=0 2024-12-02T22:15:27,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742270_1446 (size=31255) 2024-12-02T22:15:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177787903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177787923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177787924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177787924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T22:15:28,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177788223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177788244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177788244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177788244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/2a98711fbccd41a4984dadeaef57d723 2024-12-02T22:15:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/8e68766fb0614817972c69d6476d4096 is 50, key is test_row_0/B:col10/1733177726898/Put/seqid=0 2024-12-02T22:15:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742271_1447 (size=12301) 2024-12-02T22:15:28,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/8e68766fb0614817972c69d6476d4096 2024-12-02T22:15:28,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a896ad3f90f405fa5b3f06db047ba80 is 50, key is test_row_0/C:col10/1733177726898/Put/seqid=0 2024-12-02T22:15:28,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742272_1448 (size=12301) 2024-12-02T22:15:28,385 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a896ad3f90f405fa5b3f06db047ba80 2024-12-02T22:15:28,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/2a98711fbccd41a4984dadeaef57d723 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723 2024-12-02T22:15:28,397 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723, entries=150, sequenceid=332, filesize=30.5 K 2024-12-02T22:15:28,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/8e68766fb0614817972c69d6476d4096 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096 2024-12-02T22:15:28,404 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096, entries=150, sequenceid=332, filesize=12.0 K 2024-12-02T22:15:28,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/5a896ad3f90f405fa5b3f06db047ba80 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80 2024-12-02T22:15:28,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80, entries=150, sequenceid=332, filesize=12.0 K 2024-12-02T22:15:28,411 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 87f06add23a346902905707af25d359f in 923ms, sequenceid=332, compaction requested=true 2024-12-02T22:15:28,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:28,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:28,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-02T22:15:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-02T22:15:28,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-02T22:15:28,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-12-02T22:15:28,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0820 sec 2024-12-02T22:15:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T22:15:28,474 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:28,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-02T22:15:28,479 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T22:15:28,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:28,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T22:15:28,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-02T22:15:28,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:28,634 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202140489174e9f4416b8806366e9ecb919_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177727580/Put/seqid=0 2024-12-02T22:15:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742273_1449 (size=12454) 2024-12-02T22:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:28,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:28,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177788780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177788786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T22:15:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177788787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177788789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177788893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177788905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177788913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177788913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:29,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202140489174e9f4416b8806366e9ecb919_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202140489174e9f4416b8806366e9ecb919_87f06add23a346902905707af25d359f 2024-12-02T22:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7d2894d24a4941028f7e58fe2a1e3676, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7d2894d24a4941028f7e58fe2a1e3676 is 175, key is test_row_0/A:col10/1733177727580/Put/seqid=0 2024-12-02T22:15:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742274_1450 (size=31255) 2024-12-02T22:15:29,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7d2894d24a4941028f7e58fe2a1e3676 2024-12-02T22:15:29,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/30b164489dc54868b671360b50d220ca is 50, key is test_row_0/B:col10/1733177727580/Put/seqid=0 2024-12-02T22:15:29,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742275_1451 (size=12301) 2024-12-02T22:15:29,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/30b164489dc54868b671360b50d220ca 2024-12-02T22:15:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63336e0d2c24a968db63df35a3fdb6a is 50, key is test_row_0/C:col10/1733177727580/Put/seqid=0 2024-12-02T22:15:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742276_1452 (size=12301) 2024-12-02T22:15:29,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63336e0d2c24a968db63df35a3fdb6a 2024-12-02T22:15:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/7d2894d24a4941028f7e58fe2a1e3676 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676 2024-12-02T22:15:29,101 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676, entries=150, sequenceid=351, filesize=30.5 K 2024-12-02T22:15:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/30b164489dc54868b671360b50d220ca as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca 2024-12-02T22:15:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T22:15:29,110 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T22:15:29,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/b63336e0d2c24a968db63df35a3fdb6a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a 2024-12-02T22:15:29,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177789114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,116 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T22:15:29,117 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 87f06add23a346902905707af25d359f in 484ms, sequenceid=351, compaction requested=true 2024-12-02T22:15:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-02T22:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-02T22:15:29,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-02T22:15:29,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 638 msec 2024-12-02T22:15:29,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 645 msec 2024-12-02T22:15:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:29,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:29,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026951ec5edfdd4edb9b4d0ddf24689d28_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:29,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742277_1453 (size=14994) 2024-12-02T22:15:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177789183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177789198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177789199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177789303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177789323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177789323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177789423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177789523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177789533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177789533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:29,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026951ec5edfdd4edb9b4d0ddf24689d28_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026951ec5edfdd4edb9b4d0ddf24689d28_87f06add23a346902905707af25d359f 2024-12-02T22:15:29,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/51a009fc9c8c4cc0ba143bf66d7337cd, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:29,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/51a009fc9c8c4cc0ba143bf66d7337cd is 175, key is test_row_0/A:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:29,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742278_1454 (size=39949) 2024-12-02T22:15:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T22:15:29,613 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:29,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-02T22:15:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:29,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:29,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:29,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:29,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:29,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177789843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177789853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177789853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:29,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:29,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/51a009fc9c8c4cc0ba143bf66d7337cd 2024-12-02T22:15:29,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177789943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:29,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/583007ba0df64e8a8c2ae9c435b6f133 is 50, key is test_row_0/B:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:29,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742279_1455 (size=12301) 2024-12-02T22:15:30,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:30,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/583007ba0df64e8a8c2ae9c435b6f133 2024-12-02T22:15:30,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177790363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:30,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/1ac607acbe294b4cb26d5e5a24862450 is 50, key is test_row_0/C:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177790373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177790373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:30,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742280_1456 (size=12301) 2024-12-02T22:15:30,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:30,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:30,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/1ac607acbe294b4cb26d5e5a24862450 2024-12-02T22:15:30,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/51a009fc9c8c4cc0ba143bf66d7337cd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd 2024-12-02T22:15:30,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd, entries=200, sequenceid=365, filesize=39.0 K 2024-12-02T22:15:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/583007ba0df64e8a8c2ae9c435b6f133 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133 2024-12-02T22:15:30,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133, entries=150, sequenceid=365, filesize=12.0 K 2024-12-02T22:15:30,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/1ac607acbe294b4cb26d5e5a24862450 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450 2024-12-02T22:15:30,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450, entries=150, sequenceid=365, filesize=12.0 K 2024-12-02T22:15:30,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 87f06add23a346902905707af25d359f in 1694ms, sequenceid=365, compaction requested=true 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:30,825 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:15:30,825 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:30,826 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62085 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:15:30,826 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:30,826 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,827 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fe59f3a95d4c4738b9bad4f22e0230f5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=60.6 K 2024-12-02T22:15:30,827 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165549 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:15:30,827 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:30,827 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,827 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=161.7 K 2024-12-02T22:15:30,827 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,827 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd] 2024-12-02T22:15:30,827 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting fe59f3a95d4c4738b9bad4f22e0230f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723549 2024-12-02T22:15:30,827 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d8366bdd6e470ea1beee9a9a5fa243, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723549 2024-12-02T22:15:30,828 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f872dd5dee4a4ab069a3d1f8e2304b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733177724651 2024-12-02T22:15:30,828 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 75faa09078e94d6e89dcc75b3912da64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733177724651 2024-12-02T22:15:30,828 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a98711fbccd41a4984dadeaef57d723, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733177726894 2024-12-02T22:15:30,828 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e68766fb0614817972c69d6476d4096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733177726894 2024-12-02T22:15:30,829 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d2894d24a4941028f7e58fe2a1e3676, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733177727565 2024-12-02T22:15:30,829 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b164489dc54868b671360b50d220ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733177727565 2024-12-02T22:15:30,829 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a009fc9c8c4cc0ba143bf66d7337cd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:30,829 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 583007ba0df64e8a8c2ae9c435b6f133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:30,842 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#384 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:30,844 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/d4414544ffee4b0cbcd8a910d70e5818 is 50, key is test_row_0/B:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:30,846 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:30,848 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120248ddaa81fadd41bda71236941ee84baa_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:30,851 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120248ddaa81fadd41bda71236941ee84baa_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:30,852 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120248ddaa81fadd41bda71236941ee84baa_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:30,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T22:15:30,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:30,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:30,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742281_1457 (size=13051) 2024-12-02T22:15:30,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742282_1458 (size=4469) 2024-12-02T22:15:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202657f54b8867445f188aa4d7aed8ed096_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177729179/Put/seqid=0 2024-12-02T22:15:30,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742283_1459 (size=12454) 2024-12-02T22:15:30,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:30,890 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202657f54b8867445f188aa4d7aed8ed096_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202657f54b8867445f188aa4d7aed8ed096_87f06add23a346902905707af25d359f 2024-12-02T22:15:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a9de58bd792f402cad11460dd13f0d5d, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:30,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a9de58bd792f402cad11460dd13f0d5d is 175, key is test_row_0/A:col10/1733177729179/Put/seqid=0 2024-12-02T22:15:30,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742284_1460 (size=31255) 2024-12-02T22:15:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177791062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177791183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,267 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/d4414544ffee4b0cbcd8a910d70e5818 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/d4414544ffee4b0cbcd8a910d70e5818 2024-12-02T22:15:31,267 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#385 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:31,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/00fd624be81a4503aa190a9b5ef89eb1 is 175, key is test_row_0/A:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:31,273 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into d4414544ffee4b0cbcd8a910d70e5818(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:31,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:31,273 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=11, startTime=1733177730825; duration=0sec 2024-12-02T22:15:31,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:31,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:31,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T22:15:31,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62085 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T22:15:31,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:31,275 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:31,275 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a60ccfd75fb546b894a0a4e458b9d0d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=60.6 K 2024-12-02T22:15:31,275 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a60ccfd75fb546b894a0a4e458b9d0d8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733177723549 2024-12-02T22:15:31,275 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f6e490d0904e4dafd4f11cf03c80f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733177724651 2024-12-02T22:15:31,275 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a896ad3f90f405fa5b3f06db047ba80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733177726894 2024-12-02T22:15:31,276 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b63336e0d2c24a968db63df35a3fdb6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733177727565 2024-12-02T22:15:31,276 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac607acbe294b4cb26d5e5a24862450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:31,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742285_1461 (size=32005) 2024-12-02T22:15:31,285 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/00fd624be81a4503aa190a9b5ef89eb1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1 2024-12-02T22:15:31,290 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:31,291 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/10495edd1fbd4ff7b9545c880029d7bc is 50, key is test_row_0/C:col10/1733177728785/Put/seqid=0 2024-12-02T22:15:31,294 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into 00fd624be81a4503aa190a9b5ef89eb1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:31,294 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:31,294 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=11, startTime=1733177730825; duration=0sec 2024-12-02T22:15:31,294 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:31,294 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:31,298 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a9de58bd792f402cad11460dd13f0d5d 2024-12-02T22:15:31,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742286_1462 (size=13051) 2024-12-02T22:15:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/aa6bdfaf3c8845828a0280f2cee131e4 is 50, key is test_row_0/B:col10/1733177729179/Put/seqid=0 2024-12-02T22:15:31,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742287_1463 (size=12301) 2024-12-02T22:15:31,316 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/aa6bdfaf3c8845828a0280f2cee131e4 2024-12-02T22:15:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/8ad817492b624060b32638e597b54c79 is 50, key is test_row_0/C:col10/1733177729179/Put/seqid=0 2024-12-02T22:15:31,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742288_1464 (size=12301) 2024-12-02T22:15:31,334 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/8ad817492b624060b32638e597b54c79 2024-12-02T22:15:31,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/a9de58bd792f402cad11460dd13f0d5d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d 2024-12-02T22:15:31,343 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d, entries=150, sequenceid=383, filesize=30.5 K 2024-12-02T22:15:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/aa6bdfaf3c8845828a0280f2cee131e4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4 2024-12-02T22:15:31,347 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4, entries=150, sequenceid=383, filesize=12.0 K 2024-12-02T22:15:31,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/8ad817492b624060b32638e597b54c79 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79 2024-12-02T22:15:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79, entries=150, sequenceid=383, filesize=12.0 K 2024-12-02T22:15:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 87f06add23a346902905707af25d359f in 501ms, sequenceid=383, compaction requested=false 2024-12-02T22:15:31,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:31,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:31,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-02T22:15:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-02T22:15:31,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T22:15:31,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-12-02T22:15:31,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7440 sec 2024-12-02T22:15:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:31,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:31,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120242c55b380e0f4057a4871fb8efb53886_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:31,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742289_1465 (size=14994) 2024-12-02T22:15:31,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:31,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120242c55b380e0f4057a4871fb8efb53886_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120242c55b380e0f4057a4871fb8efb53886_87f06add23a346902905707af25d359f 2024-12-02T22:15:31,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/130a66099b3f44de8789f3b30de0c463, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:31,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/130a66099b3f44de8789f3b30de0c463 is 175, key is test_row_0/A:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:31,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742290_1466 (size=39949) 2024-12-02T22:15:31,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177791444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177791446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177791457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177791464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177791573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177791573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177791574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177791583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,705 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/10495edd1fbd4ff7b9545c880029d7bc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/10495edd1fbd4ff7b9545c880029d7bc 2024-12-02T22:15:31,709 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into 10495edd1fbd4ff7b9545c880029d7bc(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:31,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:31,709 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=11, startTime=1733177730825; duration=0sec 2024-12-02T22:15:31,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:31,709 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T22:15:31,764 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:31,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-02T22:15:31,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:31,767 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:31,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37506 deadline: 1733177791793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177791793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,799 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 87f06add23a346902905707af25d359f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177791793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177791793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177791793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:31,823 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/130a66099b3f44de8789f3b30de0c463 2024-12-02T22:15:31,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/4973cc791eed4a1d8ffe580607a6cf7e is 50, key is test_row_0/B:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742291_1467 (size=12301) 2024-12-02T22:15:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:31,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:31,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:31,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177792103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177792113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177792113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177792113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/4973cc791eed4a1d8ffe580607a6cf7e 2024-12-02T22:15:32,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f3ea04cb1b6d447e8f100f2cf7c80e2d is 50, key is test_row_0/C:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:32,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742292_1468 (size=12301) 2024-12-02T22:15:32,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:32,418 INFO [Thread-1670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1670. 2024-12-02T22:15:32,418 DEBUG [Thread-1670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:32,418 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,418 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:32,422 INFO [Thread-1672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1672. 2024-12-02T22:15:32,422 DEBUG [Thread-1672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:32,422 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:32,422 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,426 INFO [Thread-1666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1666. 2024-12-02T22:15:32,426 DEBUG [Thread-1666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:32,426 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,426 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:32,431 INFO [Thread-1668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1668. 2024-12-02T22:15:32,432 DEBUG [Thread-1668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:32,432 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,432 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:32,432 INFO [Thread-1674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1674. 2024-12-02T22:15:32,432 DEBUG [Thread-1674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:32,432 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,432 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:32,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:32,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:32,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37526 deadline: 1733177792623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37538 deadline: 1733177792633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37548 deadline: 1733177792633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37516 deadline: 1733177792633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:15:32,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f3ea04cb1b6d447e8f100f2cf7c80e2d 2024-12-02T22:15:32,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/130a66099b3f44de8789f3b30de0c463 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463 2024-12-02T22:15:32,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463, entries=200, sequenceid=400, filesize=39.0 K 2024-12-02T22:15:32,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/4973cc791eed4a1d8ffe580607a6cf7e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e 2024-12-02T22:15:32,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e, entries=150, sequenceid=400, filesize=12.0 K 2024-12-02T22:15:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/f3ea04cb1b6d447e8f100f2cf7c80e2d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d 2024-12-02T22:15:32,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d, entries=150, sequenceid=400, filesize=12.0 K 2024-12-02T22:15:32,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 87f06add23a346902905707af25d359f in 1270ms, sequenceid=400, compaction requested=true 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 87f06add23a346902905707af25d359f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:32,659 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:32,659 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/A is initiating minor compaction (all files) 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/B is initiating minor compaction (all files) 2024-12-02T22:15:32,660 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/A in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,660 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/B in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,660 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=100.8 K 2024-12-02T22:15:32,660 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/d4414544ffee4b0cbcd8a910d70e5818, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=36.8 K 2024-12-02T22:15:32,660 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463] 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00fd624be81a4503aa190a9b5ef89eb1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:32,660 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d4414544ffee4b0cbcd8a910d70e5818, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:32,661 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting aa6bdfaf3c8845828a0280f2cee131e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733177729179 2024-12-02T22:15:32,661 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9de58bd792f402cad11460dd13f0d5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733177729179 2024-12-02T22:15:32,661 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4973cc791eed4a1d8ffe580607a6cf7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733177731030 2024-12-02T22:15:32,661 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130a66099b3f44de8789f3b30de0c463, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733177731030 2024-12-02T22:15:32,665 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:32,667 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202c9ea2fde342543a2994d8bb4defdd27e_87f06add23a346902905707af25d359f store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:32,667 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:32,668 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/3344d626fbb340c08aaef49fadf9a6aa is 50, key is test_row_0/B:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:32,672 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202c9ea2fde342543a2994d8bb4defdd27e_87f06add23a346902905707af25d359f, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:32,672 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c9ea2fde342543a2994d8bb4defdd27e_87f06add23a346902905707af25d359f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:32,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742293_1469 (size=13153) 2024-12-02T22:15:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742294_1470 (size=4469) 2024-12-02T22:15:32,681 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/3344d626fbb340c08aaef49fadf9a6aa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/3344d626fbb340c08aaef49fadf9a6aa 2024-12-02T22:15:32,682 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#A#compaction#393 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:32,683 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/8c0d0f292fe34c4b9ea35e3cd7418d5d is 175, key is test_row_0/A:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:32,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T22:15:32,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:15:32,687 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/B of 87f06add23a346902905707af25d359f into 3344d626fbb340c08aaef49fadf9a6aa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:32,687 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:32,687 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/B, priority=13, startTime=1733177732659; duration=0sec 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:32,687 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:32,687 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:B 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:32,687 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:32,688 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:32,688 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 87f06add23a346902905707af25d359f/C is initiating minor compaction (all files) 2024-12-02T22:15:32,688 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 87f06add23a346902905707af25d359f/C in TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:32,688 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/10495edd1fbd4ff7b9545c880029d7bc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp, totalSize=36.8 K 2024-12-02T22:15:32,689 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 10495edd1fbd4ff7b9545c880029d7bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733177728773 2024-12-02T22:15:32,689 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad817492b624060b32638e597b54c79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733177729179 2024-12-02T22:15:32,690 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ea04cb1b6d447e8f100f2cf7c80e2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733177731030 2024-12-02T22:15:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742295_1471 (size=32107) 2024-12-02T22:15:32,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/8c0d0f292fe34c4b9ea35e3cd7418d5d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/8c0d0f292fe34c4b9ea35e3cd7418d5d 2024-12-02T22:15:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b00c7b3dfadf4e569bbcb8e0c6dca4d8_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177731455/Put/seqid=0 2024-12-02T22:15:32,702 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/A of 87f06add23a346902905707af25d359f into 8c0d0f292fe34c4b9ea35e3cd7418d5d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:32,702 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:32,703 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/A, priority=13, startTime=1733177732659; duration=0sec 2024-12-02T22:15:32,703 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:32,703 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:A 2024-12-02T22:15:32,714 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 87f06add23a346902905707af25d359f#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T22:15:32,715 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/97ffb8a036e84e909d77c3eb52dd9f30 is 50, key is test_row_0/C:col10/1733177731030/Put/seqid=0 2024-12-02T22:15:32,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742296_1472 (size=12454) 2024-12-02T22:15:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742297_1473 (size=13153) 2024-12-02T22:15:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:33,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b00c7b3dfadf4e569bbcb8e0c6dca4d8_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b00c7b3dfadf4e569bbcb8e0c6dca4d8_87f06add23a346902905707af25d359f 2024-12-02T22:15:33,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee is 175, key is test_row_0/A:col10/1733177731455/Put/seqid=0 2024-12-02T22:15:33,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742298_1474 (size=31255) 2024-12-02T22:15:33,125 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/97ffb8a036e84e909d77c3eb52dd9f30 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/97ffb8a036e84e909d77c3eb52dd9f30 2024-12-02T22:15:33,129 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 87f06add23a346902905707af25d359f/C of 87f06add23a346902905707af25d359f into 97ffb8a036e84e909d77c3eb52dd9f30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:33,129 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:33,129 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., storeName=87f06add23a346902905707af25d359f/C, priority=13, startTime=1733177732659; duration=0sec 2024-12-02T22:15:33,130 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:33,130 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 87f06add23a346902905707af25d359f:C 2024-12-02T22:15:33,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee 2024-12-02T22:15:33,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/ab1722e1a81747faa1c05e4360a218f7 is 50, key is test_row_0/B:col10/1733177731455/Put/seqid=0 2024-12-02T22:15:33,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742299_1475 (size=12301) 2024-12-02T22:15:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 87f06add23a346902905707af25d359f 2024-12-02T22:15:33,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. as already flushing 2024-12-02T22:15:33,645 INFO [Thread-1661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1661. 2024-12-02T22:15:33,645 INFO [Thread-1659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1659. 2024-12-02T22:15:33,645 INFO [Thread-1657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1657. 2024-12-02T22:15:33,645 DEBUG [Thread-1661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:33,645 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,645 DEBUG [Thread-1659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:33,645 DEBUG [Thread-1657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:33,645 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,645 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,645 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,646 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,646 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:33,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:33,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:33,647 INFO [Thread-1655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1655. 2024-12-02T22:15:33,647 DEBUG [Thread-1655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:33,647 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,647 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:33,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:33,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/ab1722e1a81747faa1c05e4360a218f7 2024-12-02T22:15:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d0e65bc2f37c4c209f55c645cb288be9 is 50, key is test_row_0/C:col10/1733177731455/Put/seqid=0 2024-12-02T22:15:33,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742300_1476 (size=12301) 2024-12-02T22:15:34,343 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d0e65bc2f37c4c209f55c645cb288be9 2024-12-02T22:15:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee 2024-12-02T22:15:34,349 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee, entries=150, sequenceid=419, filesize=30.5 K 2024-12-02T22:15:34,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/ab1722e1a81747faa1c05e4360a218f7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/ab1722e1a81747faa1c05e4360a218f7 2024-12-02T22:15:34,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/ab1722e1a81747faa1c05e4360a218f7, entries=150, sequenceid=419, filesize=12.0 K 2024-12-02T22:15:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/d0e65bc2f37c4c209f55c645cb288be9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d0e65bc2f37c4c209f55c645cb288be9 2024-12-02T22:15:34,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d0e65bc2f37c4c209f55c645cb288be9, entries=150, sequenceid=419, filesize=12.0 K 2024-12-02T22:15:34,358 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 87f06add23a346902905707af25d359f in 1672ms, sequenceid=419, compaction requested=false 2024-12-02T22:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 87f06add23a346902905707af25d359f: 2024-12-02T22:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-02T22:15:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-02T22:15:34,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-02T22:15:34,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5920 sec 2024-12-02T22:15:34,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5970 sec 2024-12-02T22:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T22:15:35,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:41,895 INFO [Thread-1663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1663. 2024-12-02T22:15:41,895 DEBUG [Thread-1663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:15:41,895 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:41,895 DEBUG [Thread-1663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:41,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 995 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2979 rows 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2976 rows 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2977 rows 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2979 rows 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 998 2024-12-02T22:15:41,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2985 rows 2024-12-02T22:15:41,896 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:15:41,896 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:15:41,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:41,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:41,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:15:41,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:15:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:41,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177741899"}]},"ts":"1733177741899"} 2024-12-02T22:15:41,901 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:15:41,901 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:15:41,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:15:41,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, UNASSIGN}] 2024-12-02T22:15:41,903 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, UNASSIGN 2024-12-02T22:15:41,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:41,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, UNASSIGN because future has completed 2024-12-02T22:15:41,905 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:15:41,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:42,057 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 87f06add23a346902905707af25d359f 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 87f06add23a346902905707af25d359f, disabling compactions & flushes 2024-12-02T22:15:42,057 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. after waiting 0 ms 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:42,057 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 87f06add23a346902905707af25d359f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=A 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:42,057 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=B 2024-12-02T22:15:42,058 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:42,058 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 87f06add23a346902905707af25d359f, store=C 2024-12-02T22:15:42,058 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:42,062 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025e4a0156f14446d5b3a4a4e8690aeef5_87f06add23a346902905707af25d359f is 50, key is test_row_0/A:col10/1733177733645/Put/seqid=0 2024-12-02T22:15:42,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742301_1477 (size=12454) 2024-12-02T22:15:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:42,466 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:42,469 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025e4a0156f14446d5b3a4a4e8690aeef5_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025e4a0156f14446d5b3a4a4e8690aeef5_87f06add23a346902905707af25d359f 2024-12-02T22:15:42,470 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/3b60b16f3dee49f0874ed25d62a8b853, store: [table=TestAcidGuarantees family=A region=87f06add23a346902905707af25d359f] 2024-12-02T22:15:42,470 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/3b60b16f3dee49f0874ed25d62a8b853 is 175, key is test_row_0/A:col10/1733177733645/Put/seqid=0 2024-12-02T22:15:42,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742302_1478 (size=31255) 2024-12-02T22:15:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:42,874 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/3b60b16f3dee49f0874ed25d62a8b853 2024-12-02T22:15:42,880 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/fae8277b322f43f590f6ddfa2c77242f is 50, key is test_row_0/B:col10/1733177733645/Put/seqid=0 2024-12-02T22:15:42,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742303_1479 (size=12301) 2024-12-02T22:15:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:43,283 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/fae8277b322f43f590f6ddfa2c77242f 2024-12-02T22:15:43,289 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/6374e31d780e43a0916fe8343581ed1d is 50, key is test_row_0/C:col10/1733177733645/Put/seqid=0 2024-12-02T22:15:43,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742304_1480 (size=12301) 2024-12-02T22:15:43,692 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/6374e31d780e43a0916fe8343581ed1d 2024-12-02T22:15:43,696 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/A/3b60b16f3dee49f0874ed25d62a8b853 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/3b60b16f3dee49f0874ed25d62a8b853 2024-12-02T22:15:43,699 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/3b60b16f3dee49f0874ed25d62a8b853, entries=150, sequenceid=429, filesize=30.5 K 2024-12-02T22:15:43,699 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/B/fae8277b322f43f590f6ddfa2c77242f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fae8277b322f43f590f6ddfa2c77242f 2024-12-02T22:15:43,702 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fae8277b322f43f590f6ddfa2c77242f, entries=150, sequenceid=429, filesize=12.0 K 2024-12-02T22:15:43,702 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/.tmp/C/6374e31d780e43a0916fe8343581ed1d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/6374e31d780e43a0916fe8343581ed1d 2024-12-02T22:15:43,705 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/6374e31d780e43a0916fe8343581ed1d, entries=150, sequenceid=429, filesize=12.0 K 2024-12-02T22:15:43,705 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 87f06add23a346902905707af25d359f in 1648ms, sequenceid=429, compaction requested=true 2024-12-02T22:15:43,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463] to archive 2024-12-02T22:15:43,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:43,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/55589633df054925ad7cae4a3897e206 2024-12-02T22:15:43,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/36d2a6bb4f3d40e48df07be7b4afd842 2024-12-02T22:15:43,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/182772f78028417a950f1f4b55d11d87 2024-12-02T22:15:43,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f0aa34144c5d4c4ab77b1525c77c1b85 2024-12-02T22:15:43,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e65691adc49f49ffa0d726023f137041 2024-12-02T22:15:43,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7aacf4ee3f8f435b935f598002765fdf 2024-12-02T22:15:43,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/f2c38aab237f4fbc9279b3ffb19892fb 2024-12-02T22:15:43,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/6783ef4f6f844c3ba5b2ebc31fd69d9d 2024-12-02T22:15:43,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/b215f38761a347129a47e2fbe1d03837 2024-12-02T22:15:43,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/1e5a025d3fa546dabeed522f6be8a8c3 2024-12-02T22:15:43,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a0c5b63dc41e417cb5c98a528e138324 2024-12-02T22:15:43,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/98a59c8574ad43c584cec60fb0ac0671 2024-12-02T22:15:43,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7a5e0b127f984a4c899c99c29bea30b4 2024-12-02T22:15:43,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/67580b8230074fd8b71f55b22dbdfc9d 2024-12-02T22:15:43,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/ea31337ca5434124b517cf99aa77bd74 2024-12-02T22:15:43,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/21d107dfcc5d4146bac194fc987ad4b3 2024-12-02T22:15:43,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/cc57f10d9574471b9d876a6320ffb878 2024-12-02T22:15:43,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a1d8366bdd6e470ea1beee9a9a5fa243 2024-12-02T22:15:43,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/71f872dd5dee4a4ab069a3d1f8e2304b 2024-12-02T22:15:43,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/2a98711fbccd41a4984dadeaef57d723 2024-12-02T22:15:43,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/7d2894d24a4941028f7e58fe2a1e3676 2024-12-02T22:15:43,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/51a009fc9c8c4cc0ba143bf66d7337cd 2024-12-02T22:15:43,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/00fd624be81a4503aa190a9b5ef89eb1 2024-12-02T22:15:43,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/a9de58bd792f402cad11460dd13f0d5d 2024-12-02T22:15:43,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/130a66099b3f44de8789f3b30de0c463 2024-12-02T22:15:43,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4986d898e4ef47fba8959fb73c47a927, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f5fff81263c14357bee21db3f05c20c8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/0cfdb5d9464449529015c851e98e409c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8a0a5aee1c054ffda538f10b0d726df6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fe59f3a95d4c4738b9bad4f22e0230f5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/d4414544ffee4b0cbcd8a910d70e5818, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e] to archive 2024-12-02T22:15:43,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:43,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/05432990d6a54ba3a589af14c2f0b47e 2024-12-02T22:15:43,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f777f25cfaa643cc89f0bd83d13d1205 2024-12-02T22:15:43,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4986d898e4ef47fba8959fb73c47a927 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4986d898e4ef47fba8959fb73c47a927 2024-12-02T22:15:43,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/374bd41d485a46e99ca9c93f51a295fa 2024-12-02T22:15:43,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/81776c654d514a98a4da4370e6245cfb 2024-12-02T22:15:43,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/497da1d9ce154e93b425ca6ac1e03dac 2024-12-02T22:15:43,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f5fff81263c14357bee21db3f05c20c8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/f5fff81263c14357bee21db3f05c20c8 2024-12-02T22:15:43,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/cbfc7622e53e4c48bd81379ebd0ad524 2024-12-02T22:15:43,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/17ff5effe6674b708332e90420e49a19 2024-12-02T22:15:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/0cfdb5d9464449529015c851e98e409c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/0cfdb5d9464449529015c851e98e409c 2024-12-02T22:15:43,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/82e195aae238449f9f4022c5e5afb619 2024-12-02T22:15:43,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/33f2a960a6384d789597142d6771edba 2024-12-02T22:15:43,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/49af0f85ebce454eac16781aa1c49245 2024-12-02T22:15:43,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8a0a5aee1c054ffda538f10b0d726df6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8a0a5aee1c054ffda538f10b0d726df6 2024-12-02T22:15:43,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/9a561b987c5148a18c4a93d6486fe7e7 2024-12-02T22:15:43,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/e7266e07a5634cdb9c2b53e04d3a773e 2024-12-02T22:15:43,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fe59f3a95d4c4738b9bad4f22e0230f5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fe59f3a95d4c4738b9bad4f22e0230f5 2024-12-02T22:15:43,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/dffb2b3d051b49bba77d61bbb8fdbabd 2024-12-02T22:15:43,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/75faa09078e94d6e89dcc75b3912da64 2024-12-02T22:15:43,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/8e68766fb0614817972c69d6476d4096 2024-12-02T22:15:43,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/30b164489dc54868b671360b50d220ca 2024-12-02T22:15:43,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/d4414544ffee4b0cbcd8a910d70e5818 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/d4414544ffee4b0cbcd8a910d70e5818 2024-12-02T22:15:43,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/583007ba0df64e8a8c2ae9c435b6f133 2024-12-02T22:15:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/aa6bdfaf3c8845828a0280f2cee131e4 2024-12-02T22:15:43,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/4973cc791eed4a1d8ffe580607a6cf7e 2024-12-02T22:15:43,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a8aa9b1fc8b347e49ce1d23aaf1340cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/35b5a063dd25426789eb97db0dc26437, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c3404874e4574258a48f1645b6501cc1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/4f496f035c314584ae99d546726afc91, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a60ccfd75fb546b894a0a4e458b9d0d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/10495edd1fbd4ff7b9545c880029d7bc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d] to archive 2024-12-02T22:15:43,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:15:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/00838939ba1f4f45aeca6febfce79190 2024-12-02T22:15:43,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63ece42bf6e41b78248d3c4dd48e10a 2024-12-02T22:15:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a8aa9b1fc8b347e49ce1d23aaf1340cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a8aa9b1fc8b347e49ce1d23aaf1340cd 2024-12-02T22:15:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/cc0910299e9443449dd136a79d2bd0b0 2024-12-02T22:15:43,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/7d5bc46d3f7a4e47852b74c48943ca1d 2024-12-02T22:15:43,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b4c2fd1ee78741b194cbcb9b1f18ea76 2024-12-02T22:15:43,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/35b5a063dd25426789eb97db0dc26437 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/35b5a063dd25426789eb97db0dc26437 2024-12-02T22:15:43,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/ac82f277edd744688d17b50362bf97b0 2024-12-02T22:15:43,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a30888f029f04ba7b589f49a5555bc2a 2024-12-02T22:15:43,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c3404874e4574258a48f1645b6501cc1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c3404874e4574258a48f1645b6501cc1 2024-12-02T22:15:43,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d30837dce25549dd9cf6ad6d76f96622 2024-12-02T22:15:43,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a361d98dab3b403983d8c5da9ab80e7e 2024-12-02T22:15:43,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a8151ff87524ce6ae1e93c06435331c 2024-12-02T22:15:43,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/4f496f035c314584ae99d546726afc91 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/4f496f035c314584ae99d546726afc91 2024-12-02T22:15:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/0120c40896bd40cba929b2e6c34537a8 2024-12-02T22:15:43,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/c744116e32504ad5bf5d2e43c28e6f22 2024-12-02T22:15:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a60ccfd75fb546b894a0a4e458b9d0d8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/a60ccfd75fb546b894a0a4e458b9d0d8 2024-12-02T22:15:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5fae4725bf034601924cc756915ecba3 2024-12-02T22:15:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f6f6e490d0904e4dafd4f11cf03c80f9 2024-12-02T22:15:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/5a896ad3f90f405fa5b3f06db047ba80 2024-12-02T22:15:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/b63336e0d2c24a968db63df35a3fdb6a 2024-12-02T22:15:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/10495edd1fbd4ff7b9545c880029d7bc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/10495edd1fbd4ff7b9545c880029d7bc 2024-12-02T22:15:43,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/1ac607acbe294b4cb26d5e5a24862450 2024-12-02T22:15:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/8ad817492b624060b32638e597b54c79 2024-12-02T22:15:43,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/f3ea04cb1b6d447e8f100f2cf7c80e2d 2024-12-02T22:15:43,774 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits/432.seqid, newMaxSeqId=432, maxSeqId=4 2024-12-02T22:15:43,775 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. 2024-12-02T22:15:43,775 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 87f06add23a346902905707af25d359f: Waiting for close lock at 1733177742057Running coprocessor pre-close hooks at 1733177742057Disabling compacts and flushes for region at 1733177742057Disabling writes for close at 1733177742057Obtaining lock to block concurrent updates at 1733177742057Preparing flush snapshotting stores in 87f06add23a346902905707af25d359f at 1733177742057Finished memstore snapshotting TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733177742058 (+1 ms)Flushing stores of TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f. at 1733177742058Flushing 87f06add23a346902905707af25d359f/A: creating writer at 1733177742058Flushing mob file 87f06add23a346902905707af25d359f/A: appending metadata at 1733177742062 (+4 ms)Flushing mob file 87f06add23a346902905707af25d359f/A: closing flushed file at 1733177742062Flushing 87f06add23a346902905707af25d359f/A: appending metadata at 1733177742470 (+408 ms)Flushing 87f06add23a346902905707af25d359f/A: closing flushed file at 1733177742470Flushing 87f06add23a346902905707af25d359f/B: creating writer at 1733177742877 (+407 ms)Flushing 87f06add23a346902905707af25d359f/B: appending metadata at 1733177742879 (+2 ms)Flushing 87f06add23a346902905707af25d359f/B: closing flushed file at 1733177742879Flushing 87f06add23a346902905707af25d359f/C: creating writer at 1733177743286 (+407 ms)Flushing 87f06add23a346902905707af25d359f/C: appending metadata at 1733177743288 (+2 ms)Flushing 87f06add23a346902905707af25d359f/C: closing flushed file at 1733177743288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93d24ee: reopening flushed file at 1733177743696 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a1c98df: reopening flushed file at 1733177743699 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29515f84: reopening flushed file at 1733177743702 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 87f06add23a346902905707af25d359f in 1648ms, sequenceid=429, compaction requested=true at 1733177743705 (+3 ms)Writing region close event to WAL at 1733177743772 (+67 ms)Running coprocessor post-close hooks at 1733177743775 (+3 ms)Closed at 1733177743775 2024-12-02T22:15:43,777 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 87f06add23a346902905707af25d359f 2024-12-02T22:15:43,777 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=87f06add23a346902905707af25d359f, regionState=CLOSED 2024-12-02T22:15:43,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:43,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T22:15:43,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 87f06add23a346902905707af25d359f, server=7ae73638d47e,41753,1733177627056 in 1.8740 sec 2024-12-02T22:15:43,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-02T22:15:43,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=87f06add23a346902905707af25d359f, UNASSIGN in 1.8780 sec 2024-12-02T22:15:43,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T22:15:43,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8810 sec 2024-12-02T22:15:43,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177743784"}]},"ts":"1733177743784"} 2024-12-02T22:15:43,785 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:15:43,785 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:15:43,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8880 sec 2024-12-02T22:15:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T22:15:44,044 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:44,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:15:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T22:15:44,046 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,048 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:44,050 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits] 2024-12-02T22:15:44,052 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/3b60b16f3dee49f0874ed25d62a8b853 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/3b60b16f3dee49f0874ed25d62a8b853 2024-12-02T22:15:44,053 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/8c0d0f292fe34c4b9ea35e3cd7418d5d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/8c0d0f292fe34c4b9ea35e3cd7418d5d 2024-12-02T22:15:44,054 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/A/e0d5d713c1ab4ea4a8c2d6d76dd086ee 2024-12-02T22:15:44,056 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/3344d626fbb340c08aaef49fadf9a6aa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/3344d626fbb340c08aaef49fadf9a6aa 2024-12-02T22:15:44,057 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/ab1722e1a81747faa1c05e4360a218f7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/ab1722e1a81747faa1c05e4360a218f7 2024-12-02T22:15:44,058 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fae8277b322f43f590f6ddfa2c77242f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/B/fae8277b322f43f590f6ddfa2c77242f 2024-12-02T22:15:44,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/6374e31d780e43a0916fe8343581ed1d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/6374e31d780e43a0916fe8343581ed1d 2024-12-02T22:15:44,060 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/97ffb8a036e84e909d77c3eb52dd9f30 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/97ffb8a036e84e909d77c3eb52dd9f30 2024-12-02T22:15:44,061 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d0e65bc2f37c4c209f55c645cb288be9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/C/d0e65bc2f37c4c209f55c645cb288be9 2024-12-02T22:15:44,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits/432.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f/recovered.edits/432.seqid 2024-12-02T22:15:44,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/87f06add23a346902905707af25d359f 2024-12-02T22:15:44,063 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:15:44,063 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:15:44,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T22:15:44,066 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020976682243dc4d368c36f4671308bc1a_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020976682243dc4d368c36f4671308bc1a_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,067 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021240ede969a04f2eb4d4dbb6b0fa7096_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021240ede969a04f2eb4d4dbb6b0fa7096_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,068 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202140489174e9f4416b8806366e9ecb919_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202140489174e9f4416b8806366e9ecb919_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f7712edb0764cfa813714cafc9b8329_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f7712edb0764cfa813714cafc9b8329_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,070 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120234caa637c093459e8abbe8ba62b9fac1_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120234caa637c093459e8abbe8ba62b9fac1_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120242c55b380e0f4057a4871fb8efb53886_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120242c55b380e0f4057a4871fb8efb53886_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025e4a0156f14446d5b3a4a4e8690aeef5_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025e4a0156f14446d5b3a4a4e8690aeef5_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202657f54b8867445f188aa4d7aed8ed096_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202657f54b8867445f188aa4d7aed8ed096_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026951ec5edfdd4edb9b4d0ddf24689d28_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026951ec5edfdd4edb9b4d0ddf24689d28_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,074 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027bb9f9fc69634e4887fb75fceac7c83d_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027bb9f9fc69634e4887fb75fceac7c83d_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028560bb73e36c4edb8c69b1a1942f99ac_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028560bb73e36c4edb8c69b1a1942f99ac_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,076 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028912be79a805482abab9b183ed40e0ba_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028912be79a805482abab9b183ed40e0ba_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,077 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029296d2eee0e244ff88abfc1f797c3075_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029296d2eee0e244ff88abfc1f797c3075_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,078 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120298645298a30947a48d6793da7a46a2cb_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120298645298a30947a48d6793da7a46a2cb_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,079 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b00c7b3dfadf4e569bbcb8e0c6dca4d8_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b00c7b3dfadf4e569bbcb8e0c6dca4d8_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,080 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b0264f5632b34d05803d40219886a1ce_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b0264f5632b34d05803d40219886a1ce_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,081 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b312617f637b4a728b2d19be4019e3f1_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b312617f637b4a728b2d19be4019e3f1_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,082 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c5969c2433684a42ab55e9299a342b05_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c5969c2433684a42ab55e9299a342b05_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,083 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f984895680044399982d62d184ff8c1e_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f984895680044399982d62d184ff8c1e_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,083 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fcb756ec5737434abbfcc5727ffd64c0_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fcb756ec5737434abbfcc5727ffd64c0_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,084 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd1127f96b0f40aab381eb9a31c9a138_87f06add23a346902905707af25d359f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd1127f96b0f40aab381eb9a31c9a138_87f06add23a346902905707af25d359f 2024-12-02T22:15:44,085 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:15:44,087 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,088 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:15:44,090 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:15:44,091 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,091 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:15:44,091 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177744091"}]},"ts":"9223372036854775807"} 2024-12-02T22:15:44,092 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:15:44,092 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 87f06add23a346902905707af25d359f, NAME => 'TestAcidGuarantees,,1733177709303.87f06add23a346902905707af25d359f.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:15:44,092 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:15:44,093 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177744092"}]},"ts":"9223372036854775807"} 2024-12-02T22:15:44,094 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:15:44,094 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-02T22:15:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T22:15:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:15:44,153 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:44,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=564 (was 596), ProcessCount=11 (was 11), AvailableMemoryMB=7294 (was 7916) 2024-12-02T22:15:44,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=564, ProcessCount=11, AvailableMemoryMB=7293 2024-12-02T22:15:44,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:15:44,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:15:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:15:44,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:15:44,175 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:44,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-12-02T22:15:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:44,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:15:44,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742305_1481 (size=1061) 2024-12-02T22:15:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:44,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:15:44,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742306_1482 (size=53) 2024-12-02T22:15:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1fbeb7002d31ceedab10535986bc0bcf, disabling compactions & flushes 2024-12-02T22:15:44,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. after waiting 0 ms 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:44,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fbeb7002d31ceedab10535986bc0bcf: Waiting for close lock at 1733177744987Disabling compacts and flushes for region at 1733177744987Disabling writes for close at 1733177744987Writing region close event to WAL at 1733177744987Closed at 1733177744987 2024-12-02T22:15:44,988 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:15:44,988 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177744988"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177744988"}]},"ts":"1733177744988"} 2024-12-02T22:15:44,989 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:15:44,990 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:15:44,990 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177744990"}]},"ts":"1733177744990"} 2024-12-02T22:15:44,991 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:15:44,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, ASSIGN}] 2024-12-02T22:15:44,992 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, ASSIGN 2024-12-02T22:15:44,993 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:15:45,143 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=1fbeb7002d31ceedab10535986bc0bcf, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:45,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, ASSIGN because future has completed 2024-12-02T22:15:45,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:15:45,298 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:45,298 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:15:45,299 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,299 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:15:45,299 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,299 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,300 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,301 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:45,302 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fbeb7002d31ceedab10535986bc0bcf columnFamilyName A 2024-12-02T22:15:45,302 DEBUG [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:45,302 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(327): Store=1fbeb7002d31ceedab10535986bc0bcf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:45,302 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,303 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:45,303 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fbeb7002d31ceedab10535986bc0bcf columnFamilyName B 2024-12-02T22:15:45,303 DEBUG [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:45,303 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(327): Store=1fbeb7002d31ceedab10535986bc0bcf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:45,303 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,304 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:15:45,304 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fbeb7002d31ceedab10535986bc0bcf columnFamilyName C 2024-12-02T22:15:45,304 DEBUG [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:15:45,305 INFO [StoreOpener-1fbeb7002d31ceedab10535986bc0bcf-1 {}] regionserver.HStore(327): Store=1fbeb7002d31ceedab10535986bc0bcf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:15:45,305 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:45,305 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,305 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,305 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,306 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,306 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,307 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:15:45,307 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,309 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:15:45,309 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 1fbeb7002d31ceedab10535986bc0bcf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65151994, jitterRate=-0.029159635305404663}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:15:45,309 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:45,310 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 1fbeb7002d31ceedab10535986bc0bcf: Running coprocessor pre-open hook at 1733177745300Writing region info on filesystem at 1733177745300Initializing all the Stores at 1733177745300Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177745300Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177745300Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177745300Cleaning up temporary data from old regions at 1733177745306 (+6 ms)Running coprocessor post-open hooks at 1733177745309 (+3 ms)Region opened successfully at 1733177745310 (+1 ms) 2024-12-02T22:15:45,310 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., pid=126, masterSystemTime=1733177745296 2024-12-02T22:15:45,312 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:45,312 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:45,312 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=1fbeb7002d31ceedab10535986bc0bcf, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:45,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:15:45,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T22:15:45,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 in 170 msec 2024-12-02T22:15:45,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-12-02T22:15:45,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, ASSIGN in 324 msec 2024-12-02T22:15:45,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:15:45,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177745317"}]},"ts":"1733177745317"} 2024-12-02T22:15:45,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:15:45,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:15:45,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-12-02T22:15:45,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:15:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T22:15:46,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:46,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a364c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d77f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,327 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368564fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,330 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a4ac52b 2024-12-02T22:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:15:46,331 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:15:46,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c76c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dad4a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef3b092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bbfbfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,338 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671d842b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,339 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6f855e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,341 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1314d211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,342 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594b40ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,343 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4625e85f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,344 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55db245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,346 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5213fa30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658e7972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,348 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404542d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,349 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34503b00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a665e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,352 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571b3e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,353 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e52936f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f45b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:15:46,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:15:46,360 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ebef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:15:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:15:46,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:15:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4596d0e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:15:46,363 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,363 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,363 DEBUG [Thread-2146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,363 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,363 DEBUG [Thread-2159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,363 DEBUG [Thread-2153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,364 DEBUG [Thread-2155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,364 DEBUG [Thread-2148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,366 DEBUG [Thread-2157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,367 DEBUG [Thread-2142 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,367 DEBUG [Thread-2142 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:46,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-02T22:15:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:46,371 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:46,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,374 DEBUG [Thread-2146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,374 DEBUG [Thread-2146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:46,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:46,381 DEBUG [Thread-2151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:15:46,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:15:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:15:46,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:15:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2] 2024-12-02T22:15:46,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/905da1c17cf345de994c0174cf9f1032 is 50, key is test_row_0/A:col10/1733177746372/Put/seqid=0 2024-12-02T22:15:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742307_1483 (size=9657) 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177746396, max timestamp=1733177746447Num uniques 859; 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177746396, max timestamp=1733177746447Num uniques 859; 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177746396, max timestamp=1733177746447Num uniques 859; 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:46,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:15:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:46,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177806471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177806472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177806476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177806478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177806479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177806583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177806593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177806593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177806593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177806593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177806793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/905da1c17cf345de994c0174cf9f1032 2024-12-02T22:15:46,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177806813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177806813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177806813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177806814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:46,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/7fbb706d8c8544028d0f88add72d8917 is 50, key is test_row_0/B:col10/1733177746372/Put/seqid=0 2024-12-02T22:15:46,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742308_1484 (size=9657) 2024-12-02T22:15:46,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/7fbb706d8c8544028d0f88add72d8917 2024-12-02T22:15:46,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/b4d07115536c4682821fa7c0e25c798e is 50, key is test_row_0/C:col10/1733177746372/Put/seqid=0 2024-12-02T22:15:46,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742309_1485 (size=9657) 2024-12-02T22:15:46,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:46,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:46,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:46,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177807113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177807123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177807123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177807123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177807123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:47,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:47,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:47,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:47,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:47,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:47,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/b4d07115536c4682821fa7c0e25c798e 2024-12-02T22:15:47,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/905da1c17cf345de994c0174cf9f1032 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032 2024-12-02T22:15:47,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032, entries=100, sequenceid=11, filesize=9.4 K 2024-12-02T22:15:47,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/7fbb706d8c8544028d0f88add72d8917 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917 2024-12-02T22:15:47,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917, entries=100, sequenceid=11, filesize=9.4 K 2024-12-02T22:15:47,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/b4d07115536c4682821fa7c0e25c798e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e 2024-12-02T22:15:47,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e, entries=100, sequenceid=11, filesize=9.4 K 2024-12-02T22:15:47,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fbeb7002d31ceedab10535986bc0bcf in 907ms, sequenceid=11, compaction requested=false 2024-12-02T22:15:47,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:47,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:47,289 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:15:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/8d072a6b7ef943c68627e9081206f3ab is 50, key is test_row_0/A:col10/1733177746476/Put/seqid=0 2024-12-02T22:15:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742310_1486 (size=12001) 2024-12-02T22:15:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177807655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177807662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177807665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177807667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177807673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/8d072a6b7ef943c68627e9081206f3ab 2024-12-02T22:15:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f3341c470404456b8cf0e2a343b1587c is 50, key is test_row_0/B:col10/1733177746476/Put/seqid=0 2024-12-02T22:15:47,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742311_1487 (size=12001) 2024-12-02T22:15:47,745 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f3341c470404456b8cf0e2a343b1587c 2024-12-02T22:15:47,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/9c14d1903ee44bf1822388b6290063d4 is 50, key is test_row_0/C:col10/1733177746476/Put/seqid=0 2024-12-02T22:15:47,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742312_1488 (size=12001) 2024-12-02T22:15:47,774 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/9c14d1903ee44bf1822388b6290063d4 2024-12-02T22:15:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/8d072a6b7ef943c68627e9081206f3ab as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab 2024-12-02T22:15:47,788 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T22:15:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177807783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f3341c470404456b8cf0e2a343b1587c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c 2024-12-02T22:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177807783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177807783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177807794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177807794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:47,813 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T22:15:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/9c14d1903ee44bf1822388b6290063d4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4 2024-12-02T22:15:47,818 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T22:15:47,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1fbeb7002d31ceedab10535986bc0bcf in 530ms, sequenceid=46, compaction requested=false 2024-12-02T22:15:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-02T22:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-02T22:15:47,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-02T22:15:47,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-12-02T22:15:47,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4570 sec 2024-12-02T22:15:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:48,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T22:15:48,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:48,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/85e00b2237e54413ab426b1774d18517 is 50, key is test_row_0/A:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742313_1489 (size=14341) 2024-12-02T22:15:48,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/85e00b2237e54413ab426b1774d18517 2024-12-02T22:15:48,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b3f6845b7c464069811d449c95030f5e is 50, key is test_row_0/B:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177808079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177808083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177808085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177808098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177808101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742314_1490 (size=12001) 2024-12-02T22:15:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177808203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177808213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177808213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177808233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177808233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,249 DEBUG [regionserver/7ae73638d47e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-12-02T22:15:48,251 WARN [regionserver/7ae73638d47e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1 2024-12-02T22:15:48,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177808423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177808423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177808423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177808453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177808453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T22:15:48,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b3f6845b7c464069811d449c95030f5e 2024-12-02T22:15:48,524 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:48,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-02T22:15:48,532 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:48,532 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:48,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:48,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ed7d947cd3774ead8385e32154c288f9 is 50, key is test_row_0/C:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742315_1491 (size=12001) 2024-12-02T22:15:48,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ed7d947cd3774ead8385e32154c288f9 2024-12-02T22:15:48,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/85e00b2237e54413ab426b1774d18517 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517 2024-12-02T22:15:48,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517, entries=200, sequenceid=59, filesize=14.0 K 2024-12-02T22:15:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b3f6845b7c464069811d449c95030f5e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e 2024-12-02T22:15:48,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T22:15:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ed7d947cd3774ead8385e32154c288f9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9 2024-12-02T22:15:48,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T22:15:48,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fbeb7002d31ceedab10535986bc0bcf in 565ms, sequenceid=59, compaction requested=true 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:48,570 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:48,570 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:48,571 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:48,571 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:15:48,571 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:48,572 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=32.9 K 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:15:48,572 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:48,572 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=35.2 K 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbb706d8c8544028d0f88add72d8917, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177746372 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905da1c17cf345de994c0174cf9f1032, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177746372 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f3341c470404456b8cf0e2a343b1587c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733177746469 2024-12-02T22:15:48,572 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d072a6b7ef943c68627e9081206f3ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733177746469 2024-12-02T22:15:48,573 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f6845b7c464069811d449c95030f5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:48,573 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e00b2237e54413ab426b1774d18517, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:48,583 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:48,584 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:48,584 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/890fb0ea96084296902e2fae7655710b is 50, key is test_row_0/B:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,584 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/892179b6b0ac44be8b297b8be29c54e3 is 50, key is test_row_0/A:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742317_1493 (size=12104) 2024-12-02T22:15:48,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742316_1492 (size=12104) 2024-12-02T22:15:48,605 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/890fb0ea96084296902e2fae7655710b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/890fb0ea96084296902e2fae7655710b 2024-12-02T22:15:48,618 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 890fb0ea96084296902e2fae7655710b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:48,618 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:48,618 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177748570; duration=0sec 2024-12-02T22:15:48,618 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:48,618 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:15:48,618 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:48,619 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:48,619 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:15:48,619 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:48,619 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=32.9 K 2024-12-02T22:15:48,619 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d07115536c4682821fa7c0e25c798e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733177746372 2024-12-02T22:15:48,620 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c14d1903ee44bf1822388b6290063d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733177746469 2024-12-02T22:15:48,620 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7d947cd3774ead8385e32154c288f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:48,625 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:48,626 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/23a7731b0b0b44fe84efb7ea6a575c27 is 50, key is test_row_0/C:col10/1733177748003/Put/seqid=0 2024-12-02T22:15:48,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742318_1494 (size=12104) 2024-12-02T22:15:48,637 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/23a7731b0b0b44fe84efb7ea6a575c27 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/23a7731b0b0b44fe84efb7ea6a575c27 2024-12-02T22:15:48,641 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into 23a7731b0b0b44fe84efb7ea6a575c27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:48,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:48,641 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177748570; duration=0sec 2024-12-02T22:15:48,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:48,641 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:15:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:48,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:48,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:48,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9ad58a1f9c8d4db3a9283eb0f16b7365 is 50, key is test_row_0/A:col10/1733177748098/Put/seqid=0 2024-12-02T22:15:48,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742319_1495 (size=12001) 2024-12-02T22:15:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:48,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177808757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177808763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177808763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177808763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177808764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177808873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177808883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:48,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177808883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,005 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/892179b6b0ac44be8b297b8be29c54e3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/892179b6b0ac44be8b297b8be29c54e3 2024-12-02T22:15:49,009 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 892179b6b0ac44be8b297b8be29c54e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:49,009 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:49,009 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177748570; duration=0sec 2024-12-02T22:15:49,009 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:49,009 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:15:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177809083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9ad58a1f9c8d4db3a9283eb0f16b7365 2024-12-02T22:15:49,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/48af8b57edc64667a38fdb09edac62a2 is 50, key is test_row_0/B:col10/1733177748098/Put/seqid=0 2024-12-02T22:15:49,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742320_1496 (size=12001) 2024-12-02T22:15:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177809103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177809103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177809283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177809283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,297 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:15:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177809403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177809423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177809423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,508 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/48af8b57edc64667a38fdb09edac62a2 2024-12-02T22:15:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/525f030f33fc4b22966845ff7b01ab56 is 50, key is test_row_0/C:col10/1733177748098/Put/seqid=0 2024-12-02T22:15:49,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742321_1497 (size=12001) 2024-12-02T22:15:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:49,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/525f030f33fc4b22966845ff7b01ab56 2024-12-02T22:15:49,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9ad58a1f9c8d4db3a9283eb0f16b7365 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365 2024-12-02T22:15:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177809923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:49,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365, entries=150, sequenceid=80, filesize=11.7 K 2024-12-02T22:15:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/48af8b57edc64667a38fdb09edac62a2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2 2024-12-02T22:15:49,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2, entries=150, sequenceid=80, filesize=11.7 K 2024-12-02T22:15:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/525f030f33fc4b22966845ff7b01ab56 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56 2024-12-02T22:15:49,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56, entries=150, sequenceid=80, filesize=11.7 K 2024-12-02T22:15:49,938 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fbeb7002d31ceedab10535986bc0bcf in 1253ms, sequenceid=80, compaction requested=false 2024-12-02T22:15:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-02T22:15:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-02T22:15:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:49,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:49,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T22:15:49,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-12-02T22:15:49,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4200 sec 2024-12-02T22:15:49,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/73fd6cfc1cae49bba60ffca4df921b17 is 50, key is test_row_0/A:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742322_1498 (size=14341) 2024-12-02T22:15:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177810045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177810046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177810153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177810163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177810292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,301 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177810303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,312 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/73fd6cfc1cae49bba60ffca4df921b17 2024-12-02T22:15:50,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f8b81f9297444b98bea66e894d2a58f4 is 50, key is test_row_0/B:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:50,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742323_1499 (size=12001) 2024-12-02T22:15:50,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f8b81f9297444b98bea66e894d2a58f4 2024-12-02T22:15:50,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/94ed128bb556497283312e2d3ca314be is 50, key is test_row_0/C:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:50,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742324_1500 (size=12001) 2024-12-02T22:15:50,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/94ed128bb556497283312e2d3ca314be 2024-12-02T22:15:50,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/73fd6cfc1cae49bba60ffca4df921b17 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17 2024-12-02T22:15:50,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17, entries=200, sequenceid=95, filesize=14.0 K 2024-12-02T22:15:50,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f8b81f9297444b98bea66e894d2a58f4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4 2024-12-02T22:15:50,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T22:15:50,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/94ed128bb556497283312e2d3ca314be as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be 2024-12-02T22:15:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177810374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177810374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T22:15:50,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1fbeb7002d31ceedab10535986bc0bcf in 447ms, sequenceid=95, compaction requested=true 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:50,388 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:50,388 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:50,389 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:50,389 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:50,389 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:15:50,389 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:15:50,389 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,390 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,390 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/892179b6b0ac44be8b297b8be29c54e3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=37.5 K 2024-12-02T22:15:50,390 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/890fb0ea96084296902e2fae7655710b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=35.3 K 2024-12-02T22:15:50,390 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 890fb0ea96084296902e2fae7655710b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:50,390 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892179b6b0ac44be8b297b8be29c54e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:50,390 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad58a1f9c8d4db3a9283eb0f16b7365, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733177748069 2024-12-02T22:15:50,390 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 48af8b57edc64667a38fdb09edac62a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733177748069 2024-12-02T22:15:50,391 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b81f9297444b98bea66e894d2a58f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:50,391 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fd6cfc1cae49bba60ffca4df921b17, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:50,401 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:50,402 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/18ac5615f95e4cc5a6d5de2c37d7b152 is 50, key is test_row_0/B:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:50,407 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:50,407 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/6183607e88ff41768f98ffee42c4f209 is 50, key is test_row_0/A:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:50,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742325_1501 (size=12207) 2024-12-02T22:15:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742326_1502 (size=12207) 2024-12-02T22:15:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T22:15:50,683 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:50,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T22:15:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:50,685 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:50,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:50,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:50,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:50,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:50,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:50,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:50,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:50,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:50,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fe8899235d13492a8ab1900e5d7802fa is 50, key is test_row_0/A:col10/1733177750043/Put/seqid=0 2024-12-02T22:15:50,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742327_1503 (size=14341) 2024-12-02T22:15:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177810741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177810751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:50,827 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/18ac5615f95e4cc5a6d5de2c37d7b152 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/18ac5615f95e4cc5a6d5de2c37d7b152 2024-12-02T22:15:50,829 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/6183607e88ff41768f98ffee42c4f209 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6183607e88ff41768f98ffee42c4f209 2024-12-02T22:15:50,832 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 18ac5615f95e4cc5a6d5de2c37d7b152(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:50,832 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:50,833 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177750388; duration=0sec 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:50,833 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 6183607e88ff41768f98ffee42c4f209(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:50,833 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177750388; duration=0sec 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:50,833 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:15:50,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:50,841 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:50,841 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:15:50,841 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:50,841 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/23a7731b0b0b44fe84efb7ea6a575c27, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=35.3 K 2024-12-02T22:15:50,841 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a7731b0b0b44fe84efb7ea6a575c27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733177747651 2024-12-02T22:15:50,842 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 525f030f33fc4b22966845ff7b01ab56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733177748069 2024-12-02T22:15:50,842 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ed128bb556497283312e2d3ca314be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:50,855 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:50,856 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/26de22079848434b8641d30ad29e1fd9 is 50, key is test_row_0/C:col10/1733177749939/Put/seqid=0 2024-12-02T22:15:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177810854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742328_1504 (size=12207) 2024-12-02T22:15:50,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177810863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:50,950 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:50,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:50,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:51,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177811073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177811073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fe8899235d13492a8ab1900e5d7802fa 2024-12-02T22:15:51,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c3bd722e75ec4c9caf0a74409cfb06cf is 50, key is test_row_0/B:col10/1733177750043/Put/seqid=0 2024-12-02T22:15:51,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742329_1505 (size=12001) 2024-12-02T22:15:51,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,270 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/26de22079848434b8641d30ad29e1fd9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/26de22079848434b8641d30ad29e1fd9 2024-12-02T22:15:51,274 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into 26de22079848434b8641d30ad29e1fd9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:51,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:51,274 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177750388; duration=0sec 2024-12-02T22:15:51,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:51,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:15:51,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177811383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177811383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c3bd722e75ec4c9caf0a74409cfb06cf 2024-12-02T22:15:51,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/a197f128e27643b4a9a07be0f6b51b65 is 50, key is test_row_0/C:col10/1733177750043/Put/seqid=0 2024-12-02T22:15:51,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742330_1506 (size=12001) 2024-12-02T22:15:51,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177811893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177811903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:51,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:51,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:51,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/a197f128e27643b4a9a07be0f6b51b65 2024-12-02T22:15:51,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fe8899235d13492a8ab1900e5d7802fa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa 2024-12-02T22:15:51,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa, entries=200, sequenceid=114, filesize=14.0 K 2024-12-02T22:15:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c3bd722e75ec4c9caf0a74409cfb06cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf 2024-12-02T22:15:51,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T22:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/a197f128e27643b4a9a07be0f6b51b65 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65 2024-12-02T22:15:51,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T22:15:51,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fbeb7002d31ceedab10535986bc0bcf in 1256ms, sequenceid=114, compaction requested=false 2024-12-02T22:15:51,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:52,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T22:15:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:52,063 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/bb46e6741eb44e54a7661d6a3ef6fa00 is 50, key is test_row_0/A:col10/1733177750710/Put/seqid=0 2024-12-02T22:15:52,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742331_1507 (size=12001) 2024-12-02T22:15:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:52,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:52,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177812379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177812385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,473 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/bb46e6741eb44e54a7661d6a3ef6fa00 2024-12-02T22:15:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f is 50, key is test_row_0/B:col10/1733177750710/Put/seqid=0 2024-12-02T22:15:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742332_1508 (size=12001) 2024-12-02T22:15:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177812503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177812503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177812723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177812723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:52,893 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f 2024-12-02T22:15:52,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/c840a0dc51a0479db166bc005fb7661a is 50, key is test_row_0/C:col10/1733177750710/Put/seqid=0 2024-12-02T22:15:52,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742333_1509 (size=12001) 2024-12-02T22:15:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177812904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,911 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177812923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,926 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:52,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177812973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:52,981 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177813043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177813043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/c840a0dc51a0479db166bc005fb7661a 2024-12-02T22:15:53,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/bb46e6741eb44e54a7661d6a3ef6fa00 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00 2024-12-02T22:15:53,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00, entries=150, sequenceid=130, filesize=11.7 K 2024-12-02T22:15:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f 2024-12-02T22:15:53,319 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f, entries=150, sequenceid=130, filesize=11.7 K 2024-12-02T22:15:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/c840a0dc51a0479db166bc005fb7661a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a 2024-12-02T22:15:53,323 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a, entries=150, sequenceid=130, filesize=11.7 K 2024-12-02T22:15:53,324 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fbeb7002d31ceedab10535986bc0bcf in 1261ms, sequenceid=130, compaction requested=true 2024-12-02T22:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T22:15:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T22:15:53,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T22:15:53,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-12-02T22:15:53,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.6450 sec 2024-12-02T22:15:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:53,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:53,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/36b8cb607eb4424ab759b9c086442b00 is 50, key is test_row_0/A:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742334_1510 (size=14541) 2024-12-02T22:15:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177813613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177813622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177813733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177813733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177813953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177813953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:53,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/36b8cb607eb4424ab759b9c086442b00 2024-12-02T22:15:53,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/0f00e368d1e649a584cbdeef1302e9cf is 50, key is test_row_0/B:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:53,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742335_1511 (size=12151) 2024-12-02T22:15:53,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/0f00e368d1e649a584cbdeef1302e9cf 2024-12-02T22:15:54,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/60acd9cc86bf410dab1f576bc171707f is 50, key is test_row_0/C:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:54,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742336_1512 (size=12151) 2024-12-02T22:15:54,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/60acd9cc86bf410dab1f576bc171707f 2024-12-02T22:15:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/36b8cb607eb4424ab759b9c086442b00 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00 2024-12-02T22:15:54,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00, entries=200, sequenceid=150, filesize=14.2 K 2024-12-02T22:15:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/0f00e368d1e649a584cbdeef1302e9cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf 2024-12-02T22:15:54,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf, entries=150, sequenceid=150, filesize=11.9 K 2024-12-02T22:15:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/60acd9cc86bf410dab1f576bc171707f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f 2024-12-02T22:15:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f, entries=150, sequenceid=150, filesize=11.9 K 2024-12-02T22:15:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1fbeb7002d31ceedab10535986bc0bcf in 468ms, sequenceid=150, compaction requested=true 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:54,040 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:54,040 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:15:54,043 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:54,043 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:54,043 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6183607e88ff41768f98ffee42c4f209, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=51.8 K 2024-12-02T22:15:54,043 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/18ac5615f95e4cc5a6d5de2c37d7b152, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=47.2 K 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6183607e88ff41768f98ffee42c4f209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:54,043 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ac5615f95e4cc5a6d5de2c37d7b152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8899235d13492a8ab1900e5d7802fa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177750030 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3bd722e75ec4c9caf0a74409cfb06cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177750043 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb46e6741eb44e54a7661d6a3ef6fa00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733177750710 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc2f0a7fcc547da80d1cf1b0fb14f6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733177750710 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f00e368d1e649a584cbdeef1302e9cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752374 2024-12-02T22:15:54,044 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b8cb607eb4424ab759b9c086442b00, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752363 2024-12-02T22:15:54,054 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:54,055 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/4e531ff7cfaa48739318be270ff6b6ab is 50, key is test_row_0/A:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:54,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742337_1513 (size=12493) 2024-12-02T22:15:54,064 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:54,067 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/397b14ec731e41248ad2a32de8501eb8 is 50, key is test_row_0/B:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:54,070 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/4e531ff7cfaa48739318be270ff6b6ab as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/4e531ff7cfaa48739318be270ff6b6ab 2024-12-02T22:15:54,076 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 4e531ff7cfaa48739318be270ff6b6ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:54,076 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:54,076 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=12, startTime=1733177754040; duration=0sec 2024-12-02T22:15:54,076 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:54,076 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:15:54,076 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:54,080 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:54,080 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:15:54,080 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:54,080 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/26de22079848434b8641d30ad29e1fd9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=47.2 K 2024-12-02T22:15:54,081 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26de22079848434b8641d30ad29e1fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733177748742 2024-12-02T22:15:54,081 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a197f128e27643b4a9a07be0f6b51b65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733177750043 2024-12-02T22:15:54,081 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c840a0dc51a0479db166bc005fb7661a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733177750710 2024-12-02T22:15:54,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742338_1514 (size=12493) 2024-12-02T22:15:54,082 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60acd9cc86bf410dab1f576bc171707f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752374 2024-12-02T22:15:54,092 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#434 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:54,093 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/abfcb8919e3840ea9a72a09f65cf2dcc is 50, key is test_row_0/C:col10/1733177753570/Put/seqid=0 2024-12-02T22:15:54,093 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/397b14ec731e41248ad2a32de8501eb8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/397b14ec731e41248ad2a32de8501eb8 2024-12-02T22:15:54,099 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 397b14ec731e41248ad2a32de8501eb8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:54,099 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:54,099 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=12, startTime=1733177754040; duration=0sec 2024-12-02T22:15:54,099 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:54,099 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:15:54,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742339_1515 (size=12493) 2024-12-02T22:15:54,114 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/abfcb8919e3840ea9a72a09f65cf2dcc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/abfcb8919e3840ea9a72a09f65cf2dcc 2024-12-02T22:15:54,120 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into abfcb8919e3840ea9a72a09f65cf2dcc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:54,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:54,120 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=12, startTime=1733177754040; duration=0sec 2024-12-02T22:15:54,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:54,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:15:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:54,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:15:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:54,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:54,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:54,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c9e37c2ce46d4f5f8bcf4ad83e241286 is 50, key is test_row_0/A:col10/1733177753620/Put/seqid=0 2024-12-02T22:15:54,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742340_1516 (size=14541) 2024-12-02T22:15:54,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c9e37c2ce46d4f5f8bcf4ad83e241286 2024-12-02T22:15:54,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/e4c88de101164bf2b95fa5486d798ab3 is 50, key is test_row_0/B:col10/1733177753620/Put/seqid=0 2024-12-02T22:15:54,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742341_1517 (size=12151) 2024-12-02T22:15:54,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177814335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177814347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177814453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177814463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177814673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177814673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/e4c88de101164bf2b95fa5486d798ab3 2024-12-02T22:15:54,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/fe831e0ca0ca4c3f8f211c6212635790 is 50, key is test_row_0/C:col10/1733177753620/Put/seqid=0 2024-12-02T22:15:54,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742342_1518 (size=12151) 2024-12-02T22:15:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T22:15:54,853 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:54,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-02T22:15:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T22:15:54,856 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:54,856 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:54,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177814933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,934 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177814943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:54,945 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T22:15:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177814993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177814993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T22:15:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:55,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:55,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/fe831e0ca0ca4c3f8f211c6212635790 2024-12-02T22:15:55,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c9e37c2ce46d4f5f8bcf4ad83e241286 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286 2024-12-02T22:15:55,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286, entries=200, sequenceid=166, filesize=14.2 K 2024-12-02T22:15:55,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/e4c88de101164bf2b95fa5486d798ab3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3 2024-12-02T22:15:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T22:15:55,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/fe831e0ca0ca4c3f8f211c6212635790 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790 2024-12-02T22:15:55,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790, entries=150, sequenceid=166, filesize=11.9 K 2024-12-02T22:15:55,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fbeb7002d31ceedab10535986bc0bcf in 852ms, sequenceid=166, compaction requested=false 2024-12-02T22:15:55,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:55,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T22:15:55,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:55,162 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/43c6a000c9114092a267d29b85b13526 is 50, key is test_row_0/A:col10/1733177754334/Put/seqid=0 2024-12-02T22:15:55,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742343_1519 (size=12151) 2024-12-02T22:15:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T22:15:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T22:15:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:55,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:55,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177815552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177815554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/43c6a000c9114092a267d29b85b13526 2024-12-02T22:15:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f5a465a9496f439f80aa25a9fc6adc85 is 50, key is test_row_0/B:col10/1733177754334/Put/seqid=0 2024-12-02T22:15:55,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742344_1520 (size=12151) 2024-12-02T22:15:55,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f5a465a9496f439f80aa25a9fc6adc85 2024-12-02T22:15:55,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1dfe98fca9ea4c33974e416379119530 is 50, key is test_row_0/C:col10/1733177754334/Put/seqid=0 2024-12-02T22:15:55,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742345_1521 (size=12151) 2024-12-02T22:15:55,606 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1dfe98fca9ea4c33974e416379119530 2024-12-02T22:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/43c6a000c9114092a267d29b85b13526 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526 2024-12-02T22:15:55,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526, entries=150, sequenceid=185, filesize=11.9 K 2024-12-02T22:15:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f5a465a9496f439f80aa25a9fc6adc85 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85 2024-12-02T22:15:55,620 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85, entries=150, sequenceid=185, filesize=11.9 K 2024-12-02T22:15:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1dfe98fca9ea4c33974e416379119530 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530 2024-12-02T22:15:55,624 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530, entries=150, sequenceid=185, filesize=11.9 K 2024-12-02T22:15:55,627 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fbeb7002d31ceedab10535986bc0bcf in 465ms, sequenceid=185, compaction requested=true 2024-12-02T22:15:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T22:15:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T22:15:55,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T22:15:55,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 773 msec 2024-12-02T22:15:55,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 780 msec 2024-12-02T22:15:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:55,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:55,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/438becc1eb9347848e9fe4d13a2d3752 is 50, key is test_row_0/A:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:55,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742346_1522 (size=16931) 2024-12-02T22:15:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177815766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177815769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177815873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177815893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T22:15:55,993 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:55,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-02T22:15:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:55,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:55,996 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:55,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:56,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/438becc1eb9347848e9fe4d13a2d3752 2024-12-02T22:15:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177816083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:56,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/34206a5b6bfb48d49f0df08435688ba7 is 50, key is test_row_0/B:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:56,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742347_1523 (size=12151) 2024-12-02T22:15:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177816103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:56,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T22:15:56,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:56,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T22:15:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177816403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:56,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177816423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:56,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T22:15:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:56,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/34206a5b6bfb48d49f0df08435688ba7 2024-12-02T22:15:56,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/cc49097aa11a479790c383c6bb3a2a0a is 50, key is test_row_0/C:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:56,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742348_1524 (size=12151) 2024-12-02T22:15:56,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/cc49097aa11a479790c383c6bb3a2a0a 2024-12-02T22:15:56,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/438becc1eb9347848e9fe4d13a2d3752 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752 2024-12-02T22:15:56,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752, entries=250, sequenceid=200, filesize=16.5 K 2024-12-02T22:15:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/34206a5b6bfb48d49f0df08435688ba7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7 2024-12-02T22:15:56,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7, entries=150, sequenceid=200, filesize=11.9 K 2024-12-02T22:15:56,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/cc49097aa11a479790c383c6bb3a2a0a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a 2024-12-02T22:15:56,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a, entries=150, sequenceid=200, filesize=11.9 K 2024-12-02T22:15:56,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1fbeb7002d31ceedab10535986bc0bcf in 852ms, sequenceid=200, compaction requested=true 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:56,532 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:56,532 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:56,533 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:56,533 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:56,533 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:15:56,533 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:15:56,534 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,534 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,534 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/397b14ec731e41248ad2a32de8501eb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=47.8 K 2024-12-02T22:15:56,534 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/4e531ff7cfaa48739318be270ff6b6ab, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=54.8 K 2024-12-02T22:15:56,534 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e531ff7cfaa48739318be270ff6b6ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752374 2024-12-02T22:15:56,535 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 397b14ec731e41248ad2a32de8501eb8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752374 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9e37c2ce46d4f5f8bcf4ad83e241286, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177753594 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c88de101164bf2b95fa5486d798ab3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177753605 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c6a000c9114092a267d29b85b13526, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733177754329 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a465a9496f439f80aa25a9fc6adc85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733177754329 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 34206a5b6bfb48d49f0df08435688ba7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:56,537 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438becc1eb9347848e9fe4d13a2d3752, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:56,546 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:56,547 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/05d6a69dfd1d4a6c973a07269cef48cf is 50, key is test_row_0/A:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:56,550 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#445 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:56,550 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/dd39dbd803a140709d433eca7f969589 is 50, key is test_row_0/B:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742349_1525 (size=12629) 2024-12-02T22:15:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742350_1526 (size=12629) 2024-12-02T22:15:56,579 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/dd39dbd803a140709d433eca7f969589 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd39dbd803a140709d433eca7f969589 2024-12-02T22:15:56,584 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into dd39dbd803a140709d433eca7f969589(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:56,584 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:56,584 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=12, startTime=1733177756532; duration=0sec 2024-12-02T22:15:56,584 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:56,584 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:15:56,584 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:15:56,585 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:15:56,585 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:15:56,585 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,585 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/abfcb8919e3840ea9a72a09f65cf2dcc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=47.8 K 2024-12-02T22:15:56,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting abfcb8919e3840ea9a72a09f65cf2dcc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733177752374 2024-12-02T22:15:56,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting fe831e0ca0ca4c3f8f211c6212635790, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733177753605 2024-12-02T22:15:56,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfe98fca9ea4c33974e416379119530, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733177754329 2024-12-02T22:15:56,586 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc49097aa11a479790c383c6bb3a2a0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:56,594 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:56,594 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/87502e7cab454b68bb59c526418af70e is 50, key is test_row_0/C:col10/1733177755678/Put/seqid=0 2024-12-02T22:15:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742351_1527 (size=12629) 2024-12-02T22:15:56,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:56,607 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:56,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/cb1401ed069a4ed5b78c4fd35dc25623 is 50, key is test_row_0/A:col10/1733177755764/Put/seqid=0 2024-12-02T22:15:56,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742352_1528 (size=12151) 2024-12-02T22:15:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:56,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:56,971 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/05d6a69dfd1d4a6c973a07269cef48cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/05d6a69dfd1d4a6c973a07269cef48cf 2024-12-02T22:15:56,975 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 05d6a69dfd1d4a6c973a07269cef48cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:56,975 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:56,975 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=12, startTime=1733177756532; duration=0sec 2024-12-02T22:15:56,975 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:56,975 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:15:57,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177816995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177816995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,006 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/87502e7cab454b68bb59c526418af70e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/87502e7cab454b68bb59c526418af70e 2024-12-02T22:15:57,010 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into 87502e7cab454b68bb59c526418af70e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:57,010 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:57,010 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=12, startTime=1733177756532; duration=0sec 2024-12-02T22:15:57,010 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:57,010 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:15:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46210 deadline: 1733177817003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,016 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/cb1401ed069a4ed5b78c4fd35dc25623 2024-12-02T22:15:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/4dcd797ba18a477799ecb8ffb953f98f is 50, key is test_row_0/B:col10/1733177755764/Put/seqid=0 2024-12-02T22:15:57,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742353_1529 (size=12151) 2024-12-02T22:15:57,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177817103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177817103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177817323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177817323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/4dcd797ba18a477799ecb8ffb953f98f 2024-12-02T22:15:57,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4a0d1fc0ad69411abc73b57bc40fbf81 is 50, key is test_row_0/C:col10/1733177755764/Put/seqid=0 2024-12-02T22:15:57,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742354_1530 (size=12151) 2024-12-02T22:15:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177817633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177817633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:57,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4a0d1fc0ad69411abc73b57bc40fbf81 2024-12-02T22:15:57,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/cb1401ed069a4ed5b78c4fd35dc25623 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623 2024-12-02T22:15:57,853 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T22:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/4dcd797ba18a477799ecb8ffb953f98f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f 2024-12-02T22:15:57,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T22:15:57,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4a0d1fc0ad69411abc73b57bc40fbf81 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81 2024-12-02T22:15:57,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T22:15:57,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 1fbeb7002d31ceedab10535986bc0bcf in 1255ms, sequenceid=218, compaction requested=false 2024-12-02T22:15:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T22:15:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T22:15:57,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T22:15:57,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-12-02T22:15:57,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8730 sec 2024-12-02T22:15:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T22:15:58,144 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:15:58,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:15:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-02T22:15:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:15:58,153 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:15:58,155 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:15:58,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:15:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:15:58,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:58,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/6ebb340ec97648f2a49d2ff3ccaec959 is 50, key is test_row_0/A:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:58,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742355_1531 (size=14541) 2024-12-02T22:15:58,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177818208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177818219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:15:58,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177818323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177818343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:15:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177818533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/6ebb340ec97648f2a49d2ff3ccaec959 2024-12-02T22:15:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177818563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/24846b40250149e6adceddaa8fdcffb9 is 50, key is test_row_0/B:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:58,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742356_1532 (size=12151) 2024-12-02T22:15:58,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:58,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:58,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:15:58,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177818853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177818883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:58,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46186 deadline: 1733177818963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,967 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1733177818973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:58,978 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in 1fbeb7002d31ceedab10535986bc0bcf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:15:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:58,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:58,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/24846b40250149e6adceddaa8fdcffb9 2024-12-02T22:15:58,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/802bfc315bd144bcab44b28d31f850e2 is 50, key is test_row_0/C:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742357_1533 (size=12151) 2024-12-02T22:15:59,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:15:59,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:15:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177819373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:15:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:59,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/802bfc315bd144bcab44b28d31f850e2 2024-12-02T22:15:59,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/6ebb340ec97648f2a49d2ff3ccaec959 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959 2024-12-02T22:15:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177819393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:15:59,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959, entries=200, sequenceid=237, filesize=14.2 K 2024-12-02T22:15:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:15:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:15:59,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/24846b40250149e6adceddaa8fdcffb9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9 2024-12-02T22:15:59,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9, entries=150, sequenceid=237, filesize=11.9 K 2024-12-02T22:15:59,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/802bfc315bd144bcab44b28d31f850e2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2 2024-12-02T22:15:59,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2, entries=150, sequenceid=237, filesize=11.9 K 2024-12-02T22:15:59,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1fbeb7002d31ceedab10535986bc0bcf in 1250ms, sequenceid=237, compaction requested=true 2024-12-02T22:15:59,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:59,411 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:59,411 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:15:59,411 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:15:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:59,411 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:15:59,411 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,412 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/05d6a69dfd1d4a6c973a07269cef48cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=38.4 K 2024-12-02T22:15:59,412 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:59,412 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:15:59,412 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,412 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd39dbd803a140709d433eca7f969589, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.1 K 2024-12-02T22:15:59,412 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d6a69dfd1d4a6c973a07269cef48cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:59,413 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting dd39dbd803a140709d433eca7f969589, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:59,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1401ed069a4ed5b78c4fd35dc25623, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733177755691 2024-12-02T22:15:59,413 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dcd797ba18a477799ecb8ffb953f98f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733177755691 2024-12-02T22:15:59,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ebb340ec97648f2a49d2ff3ccaec959, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:15:59,413 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 24846b40250149e6adceddaa8fdcffb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:15:59,420 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:59,420 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/5acde348b78e44458bb919458f9baeb0 is 50, key is test_row_0/B:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:59,421 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:59,422 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/b4759bfd4fe24b8b97039608fe20db73 is 50, key is test_row_0/A:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742358_1534 (size=12731) 2024-12-02T22:15:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742359_1535 (size=12731) 2024-12-02T22:15:59,434 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/5acde348b78e44458bb919458f9baeb0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/5acde348b78e44458bb919458f9baeb0 2024-12-02T22:15:59,439 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 5acde348b78e44458bb919458f9baeb0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:59,439 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:59,439 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177759411; duration=0sec 2024-12-02T22:15:59,439 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:15:59,439 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:15:59,439 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:15:59,440 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/b4759bfd4fe24b8b97039608fe20db73 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b4759bfd4fe24b8b97039608fe20db73 2024-12-02T22:15:59,440 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:15:59,440 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:15:59,440 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,440 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/87502e7cab454b68bb59c526418af70e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.1 K 2024-12-02T22:15:59,441 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 87502e7cab454b68bb59c526418af70e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733177755528 2024-12-02T22:15:59,441 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0d1fc0ad69411abc73b57bc40fbf81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733177755691 2024-12-02T22:15:59,441 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 802bfc315bd144bcab44b28d31f850e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:15:59,443 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into b4759bfd4fe24b8b97039608fe20db73(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:59,443 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:59,443 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177759410; duration=0sec 2024-12-02T22:15:59,443 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:59,443 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:15:59,449 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:15:59,449 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/d7cd5c9cb2f84d029eab38952237dc58 is 50, key is test_row_0/C:col10/1733177756994/Put/seqid=0 2024-12-02T22:15:59,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742360_1536 (size=12731) 2024-12-02T22:15:59,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:15:59,531 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:15:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:15:59,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/b9c21533763f4c07a1b3809823925d4c is 50, key is test_row_0/A:col10/1733177758218/Put/seqid=0 2024-12-02T22:15:59,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742361_1537 (size=12151) 2024-12-02T22:15:59,540 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/b9c21533763f4c07a1b3809823925d4c 2024-12-02T22:15:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/2e841a484fd947f0874f9937667f5515 is 50, key is test_row_0/B:col10/1733177758218/Put/seqid=0 2024-12-02T22:15:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742362_1538 (size=12151) 2024-12-02T22:15:59,860 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/d7cd5c9cb2f84d029eab38952237dc58 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/d7cd5c9cb2f84d029eab38952237dc58 2024-12-02T22:15:59,864 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into d7cd5c9cb2f84d029eab38952237dc58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:15:59,864 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:15:59,864 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177759411; duration=0sec 2024-12-02T22:15:59,864 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:15:59,864 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:15:59,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/2e841a484fd947f0874f9937667f5515 2024-12-02T22:15:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/e1840087ef054a88984f2012151527e1 is 50, key is test_row_0/C:col10/1733177758218/Put/seqid=0 2024-12-02T22:15:59,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742363_1539 (size=12151) 2024-12-02T22:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:16:00,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/e1840087ef054a88984f2012151527e1 2024-12-02T22:16:00,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/b9c21533763f4c07a1b3809823925d4c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c 2024-12-02T22:16:00,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c, entries=150, sequenceid=254, filesize=11.9 K 2024-12-02T22:16:00,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/2e841a484fd947f0874f9937667f5515 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515 2024-12-02T22:16:00,375 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515, entries=150, sequenceid=254, filesize=11.9 K 2024-12-02T22:16:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/e1840087ef054a88984f2012151527e1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1 2024-12-02T22:16:00,380 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1, entries=150, sequenceid=254, filesize=11.9 K 2024-12-02T22:16:00,380 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 1fbeb7002d31ceedab10535986bc0bcf in 849ms, sequenceid=254, compaction requested=false 2024-12-02T22:16:00,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-02T22:16:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-02T22:16:00,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T22:16:00,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2260 sec 2024-12-02T22:16:00,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.2400 sec 2024-12-02T22:16:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:00,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:00,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fbff87e6ee0b4313a96939c943c67079 is 50, key is test_row_0/A:col10/1733177760414/Put/seqid=0 2024-12-02T22:16:00,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742364_1540 (size=14741) 2024-12-02T22:16:00,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177760428, max timestamp=1733177760522Num uniques 859; 2024-12-02T22:16:00,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:00,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177760428, max timestamp=1733177760522Num uniques 859; 2024-12-02T22:16:00,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:00,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177760428, max timestamp=1733177760522Num uniques 859; 2024-12-02T22:16:00,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177820600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:00,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177820614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177820723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177820733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:00,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fbff87e6ee0b4313a96939c943c67079 2024-12-02T22:16:00,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/d5113407c9e4412eb92f9a2dc23c29ca is 50, key is test_row_0/B:col10/1733177760414/Put/seqid=0 2024-12-02T22:16:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742365_1541 (size=12301) 2024-12-02T22:16:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177820933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177820953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:01,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/d5113407c9e4412eb92f9a2dc23c29ca 2024-12-02T22:16:01,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/374e2041b60c4bc78529cab50dd18cbe is 50, key is test_row_0/C:col10/1733177760414/Put/seqid=0 2024-12-02T22:16:01,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742366_1542 (size=12301) 2024-12-02T22:16:01,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/374e2041b60c4bc78529cab50dd18cbe 2024-12-02T22:16:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/fbff87e6ee0b4313a96939c943c67079 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079 2024-12-02T22:16:01,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079, entries=200, sequenceid=265, filesize=14.4 K 2024-12-02T22:16:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177821243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:01,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/d5113407c9e4412eb92f9a2dc23c29ca as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca 2024-12-02T22:16:01,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca, entries=150, sequenceid=265, filesize=12.0 K 2024-12-02T22:16:01,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/374e2041b60c4bc78529cab50dd18cbe as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe 2024-12-02T22:16:01,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe, entries=150, sequenceid=265, filesize=12.0 K 2024-12-02T22:16:01,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fbeb7002d31ceedab10535986bc0bcf in 852ms, sequenceid=265, compaction requested=true 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:01,267 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:01,267 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:01,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:01,268 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:16:01,268 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:01,268 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b4759bfd4fe24b8b97039608fe20db73, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=38.7 K 2024-12-02T22:16:01,268 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4759bfd4fe24b8b97039608fe20db73, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:16:01,269 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:01,269 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/5acde348b78e44458bb919458f9baeb0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.3 K 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c21533763f4c07a1b3809823925d4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733177758193 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5acde348b78e44458bb919458f9baeb0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbff87e6ee0b4313a96939c943c67079, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733177760395 2024-12-02T22:16:01,269 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e841a484fd947f0874f9937667f5515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733177758193 2024-12-02T22:16:01,270 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d5113407c9e4412eb92f9a2dc23c29ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733177760395 2024-12-02T22:16:01,279 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:01,279 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:01,279 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/514add8ab70f46d78012d1742c61340d is 50, key is test_row_0/A:col10/1733177760414/Put/seqid=0 2024-12-02T22:16:01,280 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/018afa6c45e345b4934cadd83a95b046 is 50, key is test_row_0/B:col10/1733177760414/Put/seqid=0 2024-12-02T22:16:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:01,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T22:16:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:01,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:01,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:01,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:01,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:01,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742367_1543 (size=12983) 2024-12-02T22:16:01,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742368_1544 (size=12983) 2024-12-02T22:16:01,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c634226682064851a969c381d4758da6 is 50, key is test_row_0/A:col10/1733177761282/Put/seqid=0 2024-12-02T22:16:01,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742369_1545 (size=14741) 2024-12-02T22:16:01,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c634226682064851a969c381d4758da6 2024-12-02T22:16:01,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ac21cfa18b8743c3acd3321518f3dae0 is 50, key is test_row_0/B:col10/1733177761282/Put/seqid=0 2024-12-02T22:16:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742370_1546 (size=12301) 2024-12-02T22:16:01,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ac21cfa18b8743c3acd3321518f3dae0 2024-12-02T22:16:01,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 is 50, key is test_row_0/C:col10/1733177761282/Put/seqid=0 2024-12-02T22:16:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742371_1547 (size=12301) 2024-12-02T22:16:01,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 2024-12-02T22:16:01,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c634226682064851a969c381d4758da6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6 2024-12-02T22:16:01,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6, entries=200, sequenceid=302, filesize=14.4 K 2024-12-02T22:16:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ac21cfa18b8743c3acd3321518f3dae0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0 2024-12-02T22:16:01,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0, entries=150, sequenceid=302, filesize=12.0 K 2024-12-02T22:16:01,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 2024-12-02T22:16:01,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6, entries=150, sequenceid=302, filesize=12.0 K 2024-12-02T22:16:01,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=33.54 KB/34350 for 1fbeb7002d31ceedab10535986bc0bcf in 77ms, sequenceid=302, compaction requested=true 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T22:16:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:01,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df761b8243974523b117df14de2434f4 is 50, key is test_row_0/A:col10/1733177761340/Put/seqid=0 2024-12-02T22:16:01,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742372_1548 (size=14741) 2024-12-02T22:16:01,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df761b8243974523b117df14de2434f4 2024-12-02T22:16:01,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/732c97540dd446b3a48d1692c5a15224 is 50, key is test_row_0/B:col10/1733177761340/Put/seqid=0 2024-12-02T22:16:01,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742373_1549 (size=12301) 2024-12-02T22:16:01,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/732c97540dd446b3a48d1692c5a15224 2024-12-02T22:16:01,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/484c3ddd65d24ce49ed1e002d1069c90 is 50, key is test_row_0/C:col10/1733177761340/Put/seqid=0 2024-12-02T22:16:01,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742374_1550 (size=12301) 2024-12-02T22:16:01,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/484c3ddd65d24ce49ed1e002d1069c90 2024-12-02T22:16:01,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df761b8243974523b117df14de2434f4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4 2024-12-02T22:16:01,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4, entries=200, sequenceid=312, filesize=14.4 K 2024-12-02T22:16:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/732c97540dd446b3a48d1692c5a15224 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224 2024-12-02T22:16:01,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T22:16:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/484c3ddd65d24ce49ed1e002d1069c90 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90 2024-12-02T22:16:01,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T22:16:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=46.96 KB/48090 for 1fbeb7002d31ceedab10535986bc0bcf in 64ms, sequenceid=312, compaction requested=true 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-12-02T22:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:01,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/3d5475294ce84e82af7a8e45e29f0d63 is 50, key is test_row_0/A:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742375_1551 (size=14741) 2024-12-02T22:16:01,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/3d5475294ce84e82af7a8e45e29f0d63 2024-12-02T22:16:01,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/eca472fb47fd4220a586e7104dc03542 is 50, key is test_row_0/B:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742376_1552 (size=12301) 2024-12-02T22:16:01,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/eca472fb47fd4220a586e7104dc03542 2024-12-02T22:16:01,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2e21aec478be4dbf9e3bb5f882ccf5e9 is 50, key is test_row_0/C:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742377_1553 (size=12301) 2024-12-02T22:16:01,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2e21aec478be4dbf9e3bb5f882ccf5e9 2024-12-02T22:16:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/3d5475294ce84e82af7a8e45e29f0d63 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63 2024-12-02T22:16:01,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63, entries=200, sequenceid=323, filesize=14.4 K 2024-12-02T22:16:01,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/eca472fb47fd4220a586e7104dc03542 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542 2024-12-02T22:16:01,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542, entries=150, sequenceid=323, filesize=12.0 K 2024-12-02T22:16:01,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2e21aec478be4dbf9e3bb5f882ccf5e9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9 2024-12-02T22:16:01,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9, entries=150, sequenceid=323, filesize=12.0 K 2024-12-02T22:16:01,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for 1fbeb7002d31ceedab10535986bc0bcf in 52ms, sequenceid=323, compaction requested=true 2024-12-02T22:16:01,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=8), splitQueue=0 2024-12-02T22:16:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=9), splitQueue=0 2024-12-02T22:16:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=10), splitQueue=0 2024-12-02T22:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:01,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:01,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:01,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:01,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:01,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:01,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9696bc60be794d93b37dbdfd95d54913 is 50, key is test_row_0/A:col10/1733177761538/Put/seqid=0 2024-12-02T22:16:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742378_1554 (size=14741) 2024-12-02T22:16:01,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9696bc60be794d93b37dbdfd95d54913 2024-12-02T22:16:01,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/73e054487d334ce6b3f9f6222526526a is 50, key is test_row_0/B:col10/1733177761538/Put/seqid=0 2024-12-02T22:16:01,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742379_1555 (size=12301) 2024-12-02T22:16:01,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/73e054487d334ce6b3f9f6222526526a 2024-12-02T22:16:01,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ea91c0215ae64c82a8305bba9c379cd4 is 50, key is test_row_0/C:col10/1733177761538/Put/seqid=0 2024-12-02T22:16:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742380_1556 (size=12301) 2024-12-02T22:16:01,691 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/514add8ab70f46d78012d1742c61340d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/514add8ab70f46d78012d1742c61340d 2024-12-02T22:16:01,694 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 514add8ab70f46d78012d1742c61340d(size=12.7 K), total size for store is 55.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:01,695 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,695 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177761267; duration=0sec 2024-12-02T22:16:01,695 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=10), splitQueue=0 2024-12-02T22:16:01,695 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:01,695 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:01,695 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 3 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:01,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:01,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction 2024-12-02T22:16:01,696 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:01,696 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.0 K 2024-12-02T22:16:01,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac21cfa18b8743c3acd3321518f3dae0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302 2024-12-02T22:16:01,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732c97540dd446b3a48d1692c5a15224, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312 2024-12-02T22:16:01,696 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca472fb47fd4220a586e7104dc03542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323 2024-12-02T22:16:01,697 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/018afa6c45e345b4934cadd83a95b046 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/018afa6c45e345b4934cadd83a95b046 2024-12-02T22:16:01,701 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 018afa6c45e345b4934cadd83a95b046(size=12.7 K), total size for store is 48.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:01,701 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,701 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177761267; duration=0sec 2024-12-02T22:16:01,702 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=9), splitQueue=0 2024-12-02T22:16:01,702 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:01,702 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T22:16:01,703 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:01,703 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/7162efa251004a46b2d00bc4967e2bd8 is 50, key is test_row_0/B:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74086 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T22:16:01,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:16:01,706 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:01,706 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/d7cd5c9cb2f84d029eab38952237dc58, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=72.3 K 2024-12-02T22:16:01,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742381_1557 (size=12404) 2024-12-02T22:16:01,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cd5c9cb2f84d029eab38952237dc58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733177756945 2024-12-02T22:16:01,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e1840087ef054a88984f2012151527e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733177758193 2024-12-02T22:16:01,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 374e2041b60c4bc78529cab50dd18cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733177760395 2024-12-02T22:16:01,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ad4a63d3bdf4967bd99bf608f5b8cf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733177760598 2024-12-02T22:16:01,708 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 484c3ddd65d24ce49ed1e002d1069c90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733177761330 2024-12-02T22:16:01,708 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e21aec478be4dbf9e3bb5f882ccf5e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733177761432 2024-12-02T22:16:01,717 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#477 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:01,718 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/db4778c205504694a6a033d3ec85416b is 50, key is test_row_0/C:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742382_1558 (size=13085) 2024-12-02T22:16:01,730 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/db4778c205504694a6a033d3ec85416b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/db4778c205504694a6a033d3ec85416b 2024-12-02T22:16:01,735 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into db4778c205504694a6a033d3ec85416b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:01,735 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:01,735 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=10, startTime=1733177761501; duration=0sec 2024-12-02T22:16:01,735 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=8), splitQueue=0 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 0 eligible, 16 blocking 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:01,736 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:01,739 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:01,739 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:16:01,739 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:01,739 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/514add8ab70f46d78012d1742c61340d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=55.9 K 2024-12-02T22:16:01,739 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 514add8ab70f46d78012d1742c61340d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733177760395 2024-12-02T22:16:01,740 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c634226682064851a969c381d4758da6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733177760596 2024-12-02T22:16:01,740 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting df761b8243974523b117df14de2434f4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733177761330 2024-12-02T22:16:01,740 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d5475294ce84e82af7a8e45e29f0d63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733177761420 2024-12-02T22:16:01,747 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:01,747 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/cb308849a95c43719e5f440b08ab628e is 50, key is test_row_0/A:col10/1733177761438/Put/seqid=0 2024-12-02T22:16:01,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742383_1559 (size=13119) 2024-12-02T22:16:01,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177761702, max timestamp=1733177761755Num uniques 859; 2024-12-02T22:16:01,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:01,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177761702, max timestamp=1733177761755Num uniques 859; 2024-12-02T22:16:01,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:01,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177761702, max timestamp=1733177761755Num uniques 859; 2024-12-02T22:16:01,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177821874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177821875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:01,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ea91c0215ae64c82a8305bba9c379cd4 2024-12-02T22:16:01,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/9696bc60be794d93b37dbdfd95d54913 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913 2024-12-02T22:16:01,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913, entries=200, sequenceid=333, filesize=14.4 K 2024-12-02T22:16:01,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/73e054487d334ce6b3f9f6222526526a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a 2024-12-02T22:16:01,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T22:16:01,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ea91c0215ae64c82a8305bba9c379cd4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4 2024-12-02T22:16:01,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T22:16:01,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fbeb7002d31ceedab10535986bc0bcf in 460ms, sequenceid=333, compaction requested=false 2024-12-02T22:16:01,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:02,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T22:16:02,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/816dbc5471fe4e43ab3dab2dc245a8c6 is 50, key is test_row_0/A:col10/1733177761873/Put/seqid=0 2024-12-02T22:16:02,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742384_1560 (size=14741) 2024-12-02T22:16:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177822033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177822045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,114 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/7162efa251004a46b2d00bc4967e2bd8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7162efa251004a46b2d00bc4967e2bd8 2024-12-02T22:16:02,118 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 7162efa251004a46b2d00bc4967e2bd8(size=12.1 K), total size for store is 36.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:02,118 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,118 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=10, startTime=1733177761501; duration=0sec 2024-12-02T22:16:02,118 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-12-02T22:16:02,118 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:02,118 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,119 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:02,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37688 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:02,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:16:02,120 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,120 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/018afa6c45e345b4934cadd83a95b046, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7162efa251004a46b2d00bc4967e2bd8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.8 K 2024-12-02T22:16:02,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018afa6c45e345b4934cadd83a95b046, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733177760395 2024-12-02T22:16:02,120 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7162efa251004a46b2d00bc4967e2bd8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733177761432 2024-12-02T22:16:02,121 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e054487d334ce6b3f9f6222526526a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733177761468 2024-12-02T22:16:02,127 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:02,128 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ad54c3252c4d4d0cb2dbe3104285b548 is 50, key is test_row_0/B:col10/1733177761538/Put/seqid=0 2024-12-02T22:16:02,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742385_1561 (size=13187) 2024-12-02T22:16:02,155 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/cb308849a95c43719e5f440b08ab628e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb308849a95c43719e5f440b08ab628e 2024-12-02T22:16:02,163 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into cb308849a95c43719e5f440b08ab628e(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:02,163 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,163 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=12, startTime=1733177761361; duration=0sec 2024-12-02T22:16:02,163 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T22:16:02,163 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:02,163 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,164 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:16:02,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,165 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177822153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177822163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T22:16:02,313 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:02,313 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-12-02T22:16:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:02,315 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:02,316 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:02,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:02,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177822373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177822373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/816dbc5471fe4e43ab3dab2dc245a8c6 2024-12-02T22:16:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:02,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c450586d6161407e85b4c976eb5fde79 is 50, key is test_row_0/B:col10/1733177761873/Put/seqid=0 2024-12-02T22:16:02,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742386_1562 (size=12301) 2024-12-02T22:16:02,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T22:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,535 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ad54c3252c4d4d0cb2dbe3104285b548 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ad54c3252c4d4d0cb2dbe3104285b548 2024-12-02T22:16:02,540 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into ad54c3252c4d4d0cb2dbe3104285b548(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:02,540 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,540 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177761361; duration=0sec 2024-12-02T22:16:02,540 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:02,540 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:02,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T22:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177822693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177822693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:02,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T22:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:02,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c450586d6161407e85b4c976eb5fde79 2024-12-02T22:16:02,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4e4df3eb6cad40249d5ce50331990347 is 50, key is test_row_0/C:col10/1733177761873/Put/seqid=0 2024-12-02T22:16:02,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742387_1563 (size=12301) 2024-12-02T22:16:02,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4e4df3eb6cad40249d5ce50331990347 2024-12-02T22:16:02,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/816dbc5471fe4e43ab3dab2dc245a8c6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6 2024-12-02T22:16:02,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6, entries=200, sequenceid=374, filesize=14.4 K 2024-12-02T22:16:02,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/c450586d6161407e85b4c976eb5fde79 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79 2024-12-02T22:16:02,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T22:16:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/4e4df3eb6cad40249d5ce50331990347 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347 2024-12-02T22:16:02,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T22:16:02,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 1fbeb7002d31ceedab10535986bc0bcf in 862ms, sequenceid=374, compaction requested=true 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:02,870 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:02,870 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:16:02,871 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,871 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb308849a95c43719e5f440b08ab628e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=41.6 K 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb308849a95c43719e5f440b08ab628e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733177761432 2024-12-02T22:16:02,871 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9696bc60be794d93b37dbdfd95d54913, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733177761468 2024-12-02T22:16:02,873 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:02,873 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:16:02,873 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,873 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/db4778c205504694a6a033d3ec85416b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.8 K 2024-12-02T22:16:02,874 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 816dbc5471fe4e43ab3dab2dc245a8c6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177761823 2024-12-02T22:16:02,874 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting db4778c205504694a6a033d3ec85416b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733177761432 2024-12-02T22:16:02,876 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ea91c0215ae64c82a8305bba9c379cd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733177761468 2024-12-02T22:16:02,876 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4df3eb6cad40249d5ce50331990347, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177761864 2024-12-02T22:16:02,882 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:02,883 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/2187f4eb8f4e490fa8f3c05fef458018 is 50, key is test_row_0/A:col10/1733177761873/Put/seqid=0 2024-12-02T22:16:02,887 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:02,887 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/a61747b8203e45e2951252d0c047ea72 is 50, key is test_row_0/C:col10/1733177761873/Put/seqid=0 2024-12-02T22:16:02,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742388_1564 (size=13221) 2024-12-02T22:16:02,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:02,929 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:02,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742389_1565 (size=13187) 2024-12-02T22:16:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/a46fb55d407242d587bf86ef0cc9b5a3 is 50, key is test_row_0/A:col10/1733177762012/Put/seqid=0 2024-12-02T22:16:02,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742390_1566 (size=12301) 2024-12-02T22:16:02,969 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/a61747b8203e45e2951252d0c047ea72 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a61747b8203e45e2951252d0c047ea72 2024-12-02T22:16:02,971 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/a46fb55d407242d587bf86ef0cc9b5a3 2024-12-02T22:16:02,979 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into a61747b8203e45e2951252d0c047ea72(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:02,979 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:02,979 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177762870; duration=0sec 2024-12-02T22:16:02,979 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:02,979 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f6f5caacd6854948b1f5f38f7b2fc544 is 50, key is test_row_0/B:col10/1733177762012/Put/seqid=0 2024-12-02T22:16:02,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742391_1567 (size=12301) 2024-12-02T22:16:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:03,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177763260, max timestamp=1733177763289Num uniques 859; 2024-12-02T22:16:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177763260, max timestamp=1733177763289Num uniques 859; 2024-12-02T22:16:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:03,332 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/2187f4eb8f4e490fa8f3c05fef458018 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/2187f4eb8f4e490fa8f3c05fef458018 2024-12-02T22:16:03,338 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 2187f4eb8f4e490fa8f3c05fef458018(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:03,338 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:03,338 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177762870; duration=0sec 2024-12-02T22:16:03,338 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:03,338 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177823351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177823361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f6f5caacd6854948b1f5f38f7b2fc544 2024-12-02T22:16:03,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2f752b67a4a84f5fad96f1444170eafa is 50, key is test_row_0/C:col10/1733177762012/Put/seqid=0 2024-12-02T22:16:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742392_1568 (size=12301) 2024-12-02T22:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177823473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177823473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177823683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177823683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:03,810 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2f752b67a4a84f5fad96f1444170eafa 2024-12-02T22:16:03,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/a46fb55d407242d587bf86ef0cc9b5a3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3 2024-12-02T22:16:03,817 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T22:16:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/f6f5caacd6854948b1f5f38f7b2fc544 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544 2024-12-02T22:16:03,821 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T22:16:03,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/2f752b67a4a84f5fad96f1444170eafa as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa 2024-12-02T22:16:03,824 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T22:16:03,825 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 1fbeb7002d31ceedab10535986bc0bcf in 896ms, sequenceid=386, compaction requested=true 2024-12-02T22:16:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-02T22:16:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-02T22:16:03,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T22:16:03,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-12-02T22:16:03,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.5160 sec 2024-12-02T22:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:04,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=145.36 KB heapSize=423.16 KB 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:04,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c66dbe0482fb4d4b92e4a593dfec7ee1 is 50, key is test_row_0/A:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742393_1569 (size=17181) 2024-12-02T22:16:04,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177824032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177824038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177824153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177824153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177824363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177824363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:04,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c66dbe0482fb4d4b92e4a593dfec7ee1 2024-12-02T22:16:04,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/35079623720348799d1f15ff08cd289d is 50, key is test_row_0/B:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742394_1570 (size=12301) 2024-12-02T22:16:04,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/35079623720348799d1f15ff08cd289d 2024-12-02T22:16:04,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/19e820c270f941929788d042bc227f05 is 50, key is test_row_0/C:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742395_1571 (size=12301) 2024-12-02T22:16:04,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/19e820c270f941929788d042bc227f05 2024-12-02T22:16:04,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/c66dbe0482fb4d4b92e4a593dfec7ee1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1 2024-12-02T22:16:04,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1, entries=250, sequenceid=423, filesize=16.8 K 2024-12-02T22:16:04,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/35079623720348799d1f15ff08cd289d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d 2024-12-02T22:16:04,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d, entries=150, sequenceid=423, filesize=12.0 K 2024-12-02T22:16:04,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/19e820c270f941929788d042bc227f05 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05 2024-12-02T22:16:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T22:16:04,463 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:04,464 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:04,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05, entries=150, sequenceid=423, filesize=12.0 K 2024-12-02T22:16:04,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~443.03 KB/453664, currentSize=26.84 KB/27480 for 1fbeb7002d31ceedab10535986bc0bcf in 451ms, sequenceid=423, compaction requested=true 2024-12-02T22:16:04,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:04,465 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:16:04,465 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:04,466 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:04,466 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:16:04,466 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:04,466 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/2187f4eb8f4e490fa8f3c05fef458018, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=41.7 K 2024-12-02T22:16:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T22:16:04,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:04,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:04,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:04,469 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2187f4eb8f4e490fa8f3c05fef458018, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177761864 2024-12-02T22:16:04,470 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a46fb55d407242d587bf86ef0cc9b5a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733177762012 2024-12-02T22:16:04,471 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66dbe0482fb4d4b92e4a593dfec7ee1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763332 2024-12-02T22:16:04,471 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:04,471 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:16:04,472 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:04,472 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ad54c3252c4d4d0cb2dbe3104285b548, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=48.9 K 2024-12-02T22:16:04,472 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ad54c3252c4d4d0cb2dbe3104285b548, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733177761468 2024-12-02T22:16:04,472 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c450586d6161407e85b4c976eb5fde79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177761864 2024-12-02T22:16:04,473 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f5caacd6854948b1f5f38f7b2fc544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733177762012 2024-12-02T22:16:04,473 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 35079623720348799d1f15ff08cd289d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763345 2024-12-02T22:16:04,483 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:04,483 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/d099a3e70f694efbafa19422d9c12939 is 50, key is test_row_0/A:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,491 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:04,491 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/8da9afde6e7a44149cad059c0b77245f is 50, key is test_row_0/B:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742396_1572 (size=13323) 2024-12-02T22:16:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742397_1573 (size=13323) 2024-12-02T22:16:04,501 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/d099a3e70f694efbafa19422d9c12939 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/d099a3e70f694efbafa19422d9c12939 2024-12-02T22:16:04,510 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into d099a3e70f694efbafa19422d9c12939(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:04,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:04,510 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177764465; duration=0sec 2024-12-02T22:16:04,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:04,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:04,510 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:04,510 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/8da9afde6e7a44149cad059c0b77245f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/8da9afde6e7a44149cad059c0b77245f 2024-12-02T22:16:04,511 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:04,511 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:16:04,511 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:04,512 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a61747b8203e45e2951252d0c047ea72, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=36.9 K 2024-12-02T22:16:04,514 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a61747b8203e45e2951252d0c047ea72, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733177761864 2024-12-02T22:16:04,514 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f752b67a4a84f5fad96f1444170eafa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733177762012 2024-12-02T22:16:04,516 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e820c270f941929788d042bc227f05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763345 2024-12-02T22:16:04,517 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 8da9afde6e7a44149cad059c0b77245f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:04,517 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:04,517 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=12, startTime=1733177764465; duration=0sec 2024-12-02T22:16:04,517 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:04,517 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:04,529 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:04,530 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/b3793c5b84c848b29e8b2eea3b4c8340 is 50, key is test_row_0/C:col10/1733177763345/Put/seqid=0 2024-12-02T22:16:04,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742398_1574 (size=13289) 2024-12-02T22:16:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T22:16:04,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:04,620 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:04,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df356337737c4f8ea38a41669e4986f5 is 50, key is test_row_1/A:col10/1733177764036/Put/seqid=0 2024-12-02T22:16:04,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742399_1575 (size=7415) 2024-12-02T22:16:04,637 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df356337737c4f8ea38a41669e4986f5 2024-12-02T22:16:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/dd6b1f27cb404c668f8f126a04ba4def is 50, key is test_row_1/B:col10/1733177764036/Put/seqid=0 2024-12-02T22:16:04,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742400_1576 (size=7415) 2024-12-02T22:16:04,662 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/dd6b1f27cb404c668f8f126a04ba4def 2024-12-02T22:16:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/90cd658511584ea09f4cacf5c7e6fde5 is 50, key is test_row_1/C:col10/1733177764036/Put/seqid=0 2024-12-02T22:16:04,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742401_1577 (size=7415) 2024-12-02T22:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:04,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:04,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177764721, max timestamp=1733177764739Num uniques 859; 2024-12-02T22:16:04,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:04,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177764721, max timestamp=1733177764739Num uniques 859; 2024-12-02T22:16:04,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:04,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177764721, max timestamp=1733177764739Num uniques 859; 2024-12-02T22:16:04,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T22:16:04,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733177764800, max timestamp=1733177764819Num uniques 1068; 2024-12-02T22:16:04,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T22:16:04,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733177764800, max timestamp=1733177764819Num uniques 1068; 2024-12-02T22:16:04,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T22:16:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733177764800, max timestamp=1733177764819Num uniques 1068; 2024-12-02T22:16:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-02T22:16:04,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733177764879, max timestamp=1733177764896Num uniques 1077; 2024-12-02T22:16:04,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T22:16:04,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733177764879, max timestamp=1733177764896Num uniques 1077; 2024-12-02T22:16:04,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733177764879, max timestamp=1733177764896Num uniques 1077; 2024-12-02T22:16:04,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T22:16:04,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T22:16:04,955 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/b3793c5b84c848b29e8b2eea3b4c8340 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b3793c5b84c848b29e8b2eea3b4c8340 2024-12-02T22:16:04,959 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into b3793c5b84c848b29e8b2eea3b4c8340(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:04,959 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:04,959 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177764465; duration=0sec 2024-12-02T22:16:04,959 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:04,959 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177764941, max timestamp=1733177764983Num uniques 1086; 2024-12-02T22:16:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T22:16:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177764941, max timestamp=1733177764983Num uniques 1086; 2024-12-02T22:16:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T22:16:04,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733177764941, max timestamp=1733177764983Num uniques 1086; 2024-12-02T22:16:04,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T22:16:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733177765038, max timestamp=1733177765060Num uniques 1145; 2024-12-02T22:16:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733177765038, max timestamp=1733177765060Num uniques 1145; 2024-12-02T22:16:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733177765038, max timestamp=1733177765060Num uniques 1145; 2024-12-02T22:16:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-02T22:16:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-02T22:16:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-02T22:16:05,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/90cd658511584ea09f4cacf5c7e6fde5 2024-12-02T22:16:05,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/df356337737c4f8ea38a41669e4986f5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5 2024-12-02T22:16:05,080 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5, entries=50, sequenceid=432, filesize=7.2 K 2024-12-02T22:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/dd6b1f27cb404c668f8f126a04ba4def as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def 2024-12-02T22:16:05,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def, entries=50, sequenceid=432, filesize=7.2 K 2024-12-02T22:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/90cd658511584ea09f4cacf5c7e6fde5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5 2024-12-02T22:16:05,088 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5, entries=50, sequenceid=432, filesize=7.2 K 2024-12-02T22:16:05,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=60.38 KB/61830 for 1fbeb7002d31ceedab10535986bc0bcf in 469ms, sequenceid=432, compaction requested=false 2024-12-02T22:16:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-02T22:16:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-02T22:16:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:05,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=67.09 KB heapSize=182.39 KB 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:05,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-02T22:16:05,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2024-12-02T22:16:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T22:16:05,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/156ac2ad127e44b995582bab28c0546b is 50, key is test_row_0/A:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 629 msec 2024-12-02T22:16:05,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742402_1578 (size=17181) 2024-12-02T22:16:05,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177825182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177825188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177825293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177825303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/156ac2ad127e44b995582bab28c0546b 2024-12-02T22:16:05,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/696cd0a0d2dc4c1a83f1cb327f0a541f is 50, key is test_row_0/B:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742403_1579 (size=12301) 2024-12-02T22:16:05,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/696cd0a0d2dc4c1a83f1cb327f0a541f 2024-12-02T22:16:05,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ffa2f98a7bad4ef78b082b18657b02af is 50, key is test_row_0/C:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177825513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177825513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742404_1580 (size=12301) 2024-12-02T22:16:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T22:16:05,603 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:05,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-12-02T22:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:05,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:05,606 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:05,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:05,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T22:16:05,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:05,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1733177825833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46228 deadline: 1733177825833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 , the old value is region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., hostname=7ae73638d47e,41753,1733177627056, seqNum=2 because the exception is null or not the one we care about 2024-12-02T22:16:05,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T22:16:05,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:05,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:05,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ffa2f98a7bad4ef78b082b18657b02af 2024-12-02T22:16:05,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/156ac2ad127e44b995582bab28c0546b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b 2024-12-02T22:16:05,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b, entries=250, sequenceid=528, filesize=16.8 K 2024-12-02T22:16:05,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/696cd0a0d2dc4c1a83f1cb327f0a541f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f 2024-12-02T22:16:05,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f, entries=150, sequenceid=528, filesize=12.0 K 2024-12-02T22:16:05,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ffa2f98a7bad4ef78b082b18657b02af as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af 2024-12-02T22:16:05,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af, entries=150, sequenceid=528, filesize=12.0 K 2024-12-02T22:16:05,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~182.34 KB/186720, currentSize=114.05 KB/116790 for 1fbeb7002d31ceedab10535986bc0bcf in 853ms, sequenceid=528, compaction requested=true 2024-12-02T22:16:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:05,944 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:05,944 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:05,944 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37919 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:05,944 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/A is initiating minor compaction (all files) 2024-12-02T22:16:05,945 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/A in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,945 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/d099a3e70f694efbafa19422d9c12939, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=37.0 K 2024-12-02T22:16:05,945 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d099a3e70f694efbafa19422d9c12939, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763345 2024-12-02T22:16:05,945 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting df356337737c4f8ea38a41669e4986f5, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733177764036 2024-12-02T22:16:05,945 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:05,945 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156ac2ad127e44b995582bab28c0546b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733177765046 2024-12-02T22:16:05,945 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/B is initiating minor compaction (all files) 2024-12-02T22:16:05,945 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/B in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,946 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/8da9afde6e7a44149cad059c0b77245f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=32.3 K 2024-12-02T22:16:05,947 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da9afde6e7a44149cad059c0b77245f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763345 2024-12-02T22:16:05,947 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6b1f27cb404c668f8f126a04ba4def, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733177764036 2024-12-02T22:16:05,947 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 696cd0a0d2dc4c1a83f1cb327f0a541f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733177765076 2024-12-02T22:16:05,954 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#A#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:05,955 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/966d82e9165b450896db2af6f34443b6 is 50, key is test_row_0/A:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,962 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:05,963 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/90009348ce9a4be9afbf8185beab89ef is 50, key is test_row_0/B:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742405_1581 (size=13425) 2024-12-02T22:16:05,977 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/966d82e9165b450896db2af6f34443b6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/966d82e9165b450896db2af6f34443b6 2024-12-02T22:16:05,981 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/A of 1fbeb7002d31ceedab10535986bc0bcf into 966d82e9165b450896db2af6f34443b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:05,981 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:05,981 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/A, priority=13, startTime=1733177765944; duration=0sec 2024-12-02T22:16:05,981 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:05,981 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:05,981 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:05,982 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:05,982 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): 1fbeb7002d31ceedab10535986bc0bcf/C is initiating minor compaction (all files) 2024-12-02T22:16:05,982 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fbeb7002d31ceedab10535986bc0bcf/C in TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:05,982 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b3793c5b84c848b29e8b2eea3b4c8340, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp, totalSize=32.2 K 2024-12-02T22:16:05,983 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3793c5b84c848b29e8b2eea3b4c8340, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733177763345 2024-12-02T22:16:05,983 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90cd658511584ea09f4cacf5c7e6fde5, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733177764036 2024-12-02T22:16:05,983 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa2f98a7bad4ef78b082b18657b02af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1733177765076 2024-12-02T22:16:05,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742406_1582 (size=13425) 2024-12-02T22:16:05,994 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fbeb7002d31ceedab10535986bc0bcf#C#compaction#502 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:05,994 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ebf17e2eeab3428cacd442511ad16d0e is 50, key is test_row_0/C:col10/1733177765086/Put/seqid=0 2024-12-02T22:16:05,995 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/90009348ce9a4be9afbf8185beab89ef as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90009348ce9a4be9afbf8185beab89ef 2024-12-02T22:16:05,999 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/B of 1fbeb7002d31ceedab10535986bc0bcf into 90009348ce9a4be9afbf8185beab89ef(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:05,999 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:05,999 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/B, priority=13, startTime=1733177765944; duration=0sec 2024-12-02T22:16:05,999 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:05,999 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:06,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742407_1583 (size=13391) 2024-12-02T22:16:06,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T22:16:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:06,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/456ce995365f4872b01a0ea6106fc075 is 50, key is test_row_0/A:col10/1733177765180/Put/seqid=0 2024-12-02T22:16:06,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742408_1584 (size=12301) 2024-12-02T22:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. as already flushing 2024-12-02T22:16:06,382 INFO [Thread-2151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2151. 2024-12-02T22:16:06,382 DEBUG [Thread-2151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,383 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,383 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,389 INFO [Thread-2157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2157. 2024-12-02T22:16:06,389 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-12-02T22:16:06,389 DEBUG [Thread-2157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,389 INFO [Thread-2146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2146. 2024-12-02T22:16:06,389 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,389 DEBUG [Thread-2157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 DEBUG [Thread-2146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,389 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,389 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,389 DEBUG [Thread-2157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,397 INFO [Thread-2153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2153. 2024-12-02T22:16:06,397 DEBUG [Thread-2153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,397 DEBUG [Thread-2153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,397 DEBUG [Thread-2153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,399 INFO [Thread-2155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2155. 2024-12-02T22:16:06,399 DEBUG [Thread-2155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,399 DEBUG [Thread-2155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,399 DEBUG [Thread-2155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,400 INFO [Thread-2159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2159. 2024-12-02T22:16:06,400 DEBUG [Thread-2159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:06,400 DEBUG [Thread-2159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,400 DEBUG [Thread-2159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:06,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:06,409 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/ebf17e2eeab3428cacd442511ad16d0e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ebf17e2eeab3428cacd442511ad16d0e 2024-12-02T22:16:06,413 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fbeb7002d31ceedab10535986bc0bcf/C of 1fbeb7002d31ceedab10535986bc0bcf into ebf17e2eeab3428cacd442511ad16d0e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:06,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:06,413 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., storeName=1fbeb7002d31ceedab10535986bc0bcf/C, priority=13, startTime=1733177765944; duration=0sec 2024-12-02T22:16:06,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:06,413 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:06,473 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/456ce995365f4872b01a0ea6106fc075 2024-12-02T22:16:06,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/90570c0b5dcc4742b6f430f9e0dc284c is 50, key is test_row_0/B:col10/1733177765180/Put/seqid=0 2024-12-02T22:16:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742409_1585 (size=12301) 2024-12-02T22:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:06,881 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/90570c0b5dcc4742b6f430f9e0dc284c 2024-12-02T22:16:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1b85a8211d5543c99060374ad2af6063 is 50, key is test_row_0/C:col10/1733177765180/Put/seqid=0 2024-12-02T22:16:06,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742410_1586 (size=12301) 2024-12-02T22:16:07,074 INFO [Thread-2148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2148. 2024-12-02T22:16:07,074 DEBUG [Thread-2148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:07,074 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:07,075 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:07,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:07,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1b85a8211d5543c99060374ad2af6063 2024-12-02T22:16:07,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/456ce995365f4872b01a0ea6106fc075 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/456ce995365f4872b01a0ea6106fc075 2024-12-02T22:16:07,296 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/456ce995365f4872b01a0ea6106fc075, entries=150, sequenceid=550, filesize=12.0 K 2024-12-02T22:16:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/90570c0b5dcc4742b6f430f9e0dc284c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90570c0b5dcc4742b6f430f9e0dc284c 2024-12-02T22:16:07,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90570c0b5dcc4742b6f430f9e0dc284c, entries=150, sequenceid=550, filesize=12.0 K 2024-12-02T22:16:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/1b85a8211d5543c99060374ad2af6063 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1b85a8211d5543c99060374ad2af6063 2024-12-02T22:16:07,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1b85a8211d5543c99060374ad2af6063, entries=150, sequenceid=550, filesize=12.0 K 2024-12-02T22:16:07,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=40.25 KB/41220 for 1fbeb7002d31ceedab10535986bc0bcf in 1238ms, sequenceid=550, compaction requested=false 2024-12-02T22:16:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-02T22:16:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-02T22:16:07,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T22:16:07,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-12-02T22:16:07,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.7020 sec 2024-12-02T22:16:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T22:16:07,763 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:09,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:09,064 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-12-02T22:16:09,064 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:09,064 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:09,064 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:09,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:09,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:09,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/ae1568e4d8a448feba675cc968b2592b is 50, key is test_row_0/A:col10/1733177769063/Put/seqid=0 2024-12-02T22:16:09,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742411_1587 (size=12301) 2024-12-02T22:16:09,074 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-12-02T22:16:09,074 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:09,074 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,074 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 208 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1136 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3407 rows 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1139 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3414 rows 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1146 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3433 rows 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1143 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3427 rows 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1138 2024-12-02T22:16:09,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3407 rows 2024-12-02T22:16:09,075 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:16:09,075 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:16:09,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:09,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:09,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:09,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177769078"}]},"ts":"1733177769078"} 2024-12-02T22:16:09,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:16:09,079 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:16:09,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:16:09,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, UNASSIGN}] 2024-12-02T22:16:09,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, UNASSIGN 2024-12-02T22:16:09,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1fbeb7002d31ceedab10535986bc0bcf, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:09,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, UNASSIGN because future has completed 2024-12-02T22:16:09,083 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:16:09,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:16:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:09,235 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:09,235 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:16:09,235 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 1fbeb7002d31ceedab10535986bc0bcf, disabling compactions & flushes 2024-12-02T22:16:09,235 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:09,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/ae1568e4d8a448feba675cc968b2592b 2024-12-02T22:16:09,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b671b917598b4ab795e810986d2579d1 is 50, key is test_row_0/B:col10/1733177769063/Put/seqid=0 2024-12-02T22:16:09,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742412_1588 (size=12301) 2024-12-02T22:16:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:09,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b671b917598b4ab795e810986d2579d1 2024-12-02T22:16:09,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/794cdc8e310345e4acc968f36b53641b is 50, key is test_row_0/C:col10/1733177769063/Put/seqid=0 2024-12-02T22:16:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742413_1589 (size=12301) 2024-12-02T22:16:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:10,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/794cdc8e310345e4acc968f36b53641b 2024-12-02T22:16:10,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/ae1568e4d8a448feba675cc968b2592b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/ae1568e4d8a448feba675cc968b2592b 2024-12-02T22:16:10,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/ae1568e4d8a448feba675cc968b2592b, entries=150, sequenceid=561, filesize=12.0 K 2024-12-02T22:16:10,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/b671b917598b4ab795e810986d2579d1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b671b917598b4ab795e810986d2579d1 2024-12-02T22:16:10,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b671b917598b4ab795e810986d2579d1, entries=150, sequenceid=561, filesize=12.0 K 2024-12-02T22:16:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/794cdc8e310345e4acc968f36b53641b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/794cdc8e310345e4acc968f36b53641b 2024-12-02T22:16:10,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/794cdc8e310345e4acc968f36b53641b, entries=150, sequenceid=561, filesize=12.0 K 2024-12-02T22:16:10,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 1fbeb7002d31ceedab10535986bc0bcf in 1239ms, sequenceid=561, compaction requested=true 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fbeb7002d31ceedab10535986bc0bcf: 2024-12-02T22:16:10,303 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:10,303 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:10,303 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. after waiting 0 ms 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:10,303 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:A 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fbeb7002d31ceedab10535986bc0bcf:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. because compaction request was cancelled 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:B 2024-12-02T22:16:10,303 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fbeb7002d31ceedab10535986bc0bcf:C 2024-12-02T22:16:10,303 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 1fbeb7002d31ceedab10535986bc0bcf 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-02T22:16:10,303 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=A 2024-12-02T22:16:10,304 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:10,304 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=B 2024-12-02T22:16:10,304 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:10,304 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fbeb7002d31ceedab10535986bc0bcf, store=C 2024-12-02T22:16:10,304 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:10,306 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/26169fba43ad4bae8c21fe63883b9f88 is 50, key is test_row_1/A:col10/1733177769073/Put/seqid=0 2024-12-02T22:16:10,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742414_1590 (size=7415) 2024-12-02T22:16:10,710 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/26169fba43ad4bae8c21fe63883b9f88 2024-12-02T22:16:10,717 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ba42a2478c0d4831a8e76c0c13a87ac8 is 50, key is test_row_1/B:col10/1733177769073/Put/seqid=0 2024-12-02T22:16:10,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742415_1591 (size=7415) 2024-12-02T22:16:11,122 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ba42a2478c0d4831a8e76c0c13a87ac8 2024-12-02T22:16:11,127 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/15624212d62f43df8db3fb3b0163f271 is 50, key is test_row_1/C:col10/1733177769073/Put/seqid=0 2024-12-02T22:16:11,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742416_1592 (size=7415) 2024-12-02T22:16:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:11,531 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/15624212d62f43df8db3fb3b0163f271 2024-12-02T22:16:11,535 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/A/26169fba43ad4bae8c21fe63883b9f88 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/26169fba43ad4bae8c21fe63883b9f88 2024-12-02T22:16:11,537 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/26169fba43ad4bae8c21fe63883b9f88, entries=50, sequenceid=565, filesize=7.2 K 2024-12-02T22:16:11,538 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/B/ba42a2478c0d4831a8e76c0c13a87ac8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ba42a2478c0d4831a8e76c0c13a87ac8 2024-12-02T22:16:11,541 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ba42a2478c0d4831a8e76c0c13a87ac8, entries=50, sequenceid=565, filesize=7.2 K 2024-12-02T22:16:11,541 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/.tmp/C/15624212d62f43df8db3fb3b0163f271 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/15624212d62f43df8db3fb3b0163f271 2024-12-02T22:16:11,544 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/15624212d62f43df8db3fb3b0163f271, entries=50, sequenceid=565, filesize=7.2 K 2024-12-02T22:16:11,544 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 1fbeb7002d31ceedab10535986bc0bcf in 1241ms, sequenceid=565, compaction requested=true 2024-12-02T22:16:11,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/892179b6b0ac44be8b297b8be29c54e3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6183607e88ff41768f98ffee42c4f209, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/4e531ff7cfaa48739318be270ff6b6ab, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/05d6a69dfd1d4a6c973a07269cef48cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b4759bfd4fe24b8b97039608fe20db73, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/514add8ab70f46d78012d1742c61340d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb308849a95c43719e5f440b08ab628e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/2187f4eb8f4e490fa8f3c05fef458018, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/d099a3e70f694efbafa19422d9c12939, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b] to archive 2024-12-02T22:16:11,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:11,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/905da1c17cf345de994c0174cf9f1032 2024-12-02T22:16:11,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/8d072a6b7ef943c68627e9081206f3ab 2024-12-02T22:16:11,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/85e00b2237e54413ab426b1774d18517 2024-12-02T22:16:11,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/892179b6b0ac44be8b297b8be29c54e3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/892179b6b0ac44be8b297b8be29c54e3 2024-12-02T22:16:11,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9ad58a1f9c8d4db3a9283eb0f16b7365 2024-12-02T22:16:11,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/73fd6cfc1cae49bba60ffca4df921b17 2024-12-02T22:16:11,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6183607e88ff41768f98ffee42c4f209 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6183607e88ff41768f98ffee42c4f209 2024-12-02T22:16:11,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fe8899235d13492a8ab1900e5d7802fa 2024-12-02T22:16:11,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/bb46e6741eb44e54a7661d6a3ef6fa00 2024-12-02T22:16:11,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/36b8cb607eb4424ab759b9c086442b00 2024-12-02T22:16:11,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/4e531ff7cfaa48739318be270ff6b6ab to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/4e531ff7cfaa48739318be270ff6b6ab 2024-12-02T22:16:11,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c9e37c2ce46d4f5f8bcf4ad83e241286 2024-12-02T22:16:11,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/43c6a000c9114092a267d29b85b13526 2024-12-02T22:16:11,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/438becc1eb9347848e9fe4d13a2d3752 2024-12-02T22:16:11,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/05d6a69dfd1d4a6c973a07269cef48cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/05d6a69dfd1d4a6c973a07269cef48cf 2024-12-02T22:16:11,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb1401ed069a4ed5b78c4fd35dc25623 2024-12-02T22:16:11,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/6ebb340ec97648f2a49d2ff3ccaec959 2024-12-02T22:16:11,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b4759bfd4fe24b8b97039608fe20db73 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b4759bfd4fe24b8b97039608fe20db73 2024-12-02T22:16:11,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/b9c21533763f4c07a1b3809823925d4c 2024-12-02T22:16:11,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/fbff87e6ee0b4313a96939c943c67079 2024-12-02T22:16:11,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/514add8ab70f46d78012d1742c61340d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/514add8ab70f46d78012d1742c61340d 2024-12-02T22:16:11,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c634226682064851a969c381d4758da6 2024-12-02T22:16:11,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df761b8243974523b117df14de2434f4 2024-12-02T22:16:11,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/3d5475294ce84e82af7a8e45e29f0d63 2024-12-02T22:16:11,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb308849a95c43719e5f440b08ab628e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/cb308849a95c43719e5f440b08ab628e 2024-12-02T22:16:11,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/9696bc60be794d93b37dbdfd95d54913 2024-12-02T22:16:11,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/816dbc5471fe4e43ab3dab2dc245a8c6 2024-12-02T22:16:11,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/2187f4eb8f4e490fa8f3c05fef458018 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/2187f4eb8f4e490fa8f3c05fef458018 2024-12-02T22:16:11,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/a46fb55d407242d587bf86ef0cc9b5a3 2024-12-02T22:16:11,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/c66dbe0482fb4d4b92e4a593dfec7ee1 2024-12-02T22:16:11,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/d099a3e70f694efbafa19422d9c12939 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/d099a3e70f694efbafa19422d9c12939 2024-12-02T22:16:11,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/df356337737c4f8ea38a41669e4986f5 2024-12-02T22:16:11,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/156ac2ad127e44b995582bab28c0546b 2024-12-02T22:16:11,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/890fb0ea96084296902e2fae7655710b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/18ac5615f95e4cc5a6d5de2c37d7b152, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/397b14ec731e41248ad2a32de8501eb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd39dbd803a140709d433eca7f969589, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/5acde348b78e44458bb919458f9baeb0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/018afa6c45e345b4934cadd83a95b046, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7162efa251004a46b2d00bc4967e2bd8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ad54c3252c4d4d0cb2dbe3104285b548, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/8da9afde6e7a44149cad059c0b77245f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f] to archive 2024-12-02T22:16:11,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:11,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7fbb706d8c8544028d0f88add72d8917 2024-12-02T22:16:11,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f3341c470404456b8cf0e2a343b1587c 2024-12-02T22:16:11,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/890fb0ea96084296902e2fae7655710b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/890fb0ea96084296902e2fae7655710b 2024-12-02T22:16:11,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b3f6845b7c464069811d449c95030f5e 2024-12-02T22:16:11,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/48af8b57edc64667a38fdb09edac62a2 2024-12-02T22:16:11,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/18ac5615f95e4cc5a6d5de2c37d7b152 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/18ac5615f95e4cc5a6d5de2c37d7b152 2024-12-02T22:16:11,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f8b81f9297444b98bea66e894d2a58f4 2024-12-02T22:16:11,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c3bd722e75ec4c9caf0a74409cfb06cf 2024-12-02T22:16:11,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ccc2f0a7fcc547da80d1cf1b0fb14f6f 2024-12-02T22:16:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/397b14ec731e41248ad2a32de8501eb8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/397b14ec731e41248ad2a32de8501eb8 2024-12-02T22:16:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/0f00e368d1e649a584cbdeef1302e9cf 2024-12-02T22:16:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/e4c88de101164bf2b95fa5486d798ab3 2024-12-02T22:16:11,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f5a465a9496f439f80aa25a9fc6adc85 2024-12-02T22:16:11,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd39dbd803a140709d433eca7f969589 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd39dbd803a140709d433eca7f969589 2024-12-02T22:16:11,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/34206a5b6bfb48d49f0df08435688ba7 2024-12-02T22:16:11,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/4dcd797ba18a477799ecb8ffb953f98f 2024-12-02T22:16:11,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/5acde348b78e44458bb919458f9baeb0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/5acde348b78e44458bb919458f9baeb0 2024-12-02T22:16:11,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/24846b40250149e6adceddaa8fdcffb9 2024-12-02T22:16:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/2e841a484fd947f0874f9937667f5515 2024-12-02T22:16:11,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/018afa6c45e345b4934cadd83a95b046 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/018afa6c45e345b4934cadd83a95b046 2024-12-02T22:16:11,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/d5113407c9e4412eb92f9a2dc23c29ca 2024-12-02T22:16:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ac21cfa18b8743c3acd3321518f3dae0 2024-12-02T22:16:11,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/732c97540dd446b3a48d1692c5a15224 2024-12-02T22:16:11,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7162efa251004a46b2d00bc4967e2bd8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/7162efa251004a46b2d00bc4967e2bd8 2024-12-02T22:16:11,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/eca472fb47fd4220a586e7104dc03542 2024-12-02T22:16:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ad54c3252c4d4d0cb2dbe3104285b548 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ad54c3252c4d4d0cb2dbe3104285b548 2024-12-02T22:16:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/73e054487d334ce6b3f9f6222526526a 2024-12-02T22:16:11,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/c450586d6161407e85b4c976eb5fde79 2024-12-02T22:16:11,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/f6f5caacd6854948b1f5f38f7b2fc544 2024-12-02T22:16:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/8da9afde6e7a44149cad059c0b77245f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/8da9afde6e7a44149cad059c0b77245f 2024-12-02T22:16:11,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/35079623720348799d1f15ff08cd289d 2024-12-02T22:16:11,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/dd6b1f27cb404c668f8f126a04ba4def 2024-12-02T22:16:11,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/696cd0a0d2dc4c1a83f1cb327f0a541f 2024-12-02T22:16:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/23a7731b0b0b44fe84efb7ea6a575c27, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/26de22079848434b8641d30ad29e1fd9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/abfcb8919e3840ea9a72a09f65cf2dcc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/87502e7cab454b68bb59c526418af70e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/d7cd5c9cb2f84d029eab38952237dc58, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/db4778c205504694a6a033d3ec85416b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a61747b8203e45e2951252d0c047ea72, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b3793c5b84c848b29e8b2eea3b4c8340, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af] to archive 2024-12-02T22:16:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:11,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b4d07115536c4682821fa7c0e25c798e 2024-12-02T22:16:11,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/9c14d1903ee44bf1822388b6290063d4 2024-12-02T22:16:11,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/23a7731b0b0b44fe84efb7ea6a575c27 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/23a7731b0b0b44fe84efb7ea6a575c27 2024-12-02T22:16:11,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ed7d947cd3774ead8385e32154c288f9 2024-12-02T22:16:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/525f030f33fc4b22966845ff7b01ab56 2024-12-02T22:16:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/26de22079848434b8641d30ad29e1fd9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/26de22079848434b8641d30ad29e1fd9 2024-12-02T22:16:11,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/94ed128bb556497283312e2d3ca314be 2024-12-02T22:16:11,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a197f128e27643b4a9a07be0f6b51b65 2024-12-02T22:16:11,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/c840a0dc51a0479db166bc005fb7661a 2024-12-02T22:16:11,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/abfcb8919e3840ea9a72a09f65cf2dcc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/abfcb8919e3840ea9a72a09f65cf2dcc 2024-12-02T22:16:11,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/60acd9cc86bf410dab1f576bc171707f 2024-12-02T22:16:11,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/fe831e0ca0ca4c3f8f211c6212635790 2024-12-02T22:16:11,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1dfe98fca9ea4c33974e416379119530 2024-12-02T22:16:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/87502e7cab454b68bb59c526418af70e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/87502e7cab454b68bb59c526418af70e 2024-12-02T22:16:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/cc49097aa11a479790c383c6bb3a2a0a 2024-12-02T22:16:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4a0d1fc0ad69411abc73b57bc40fbf81 2024-12-02T22:16:11,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/d7cd5c9cb2f84d029eab38952237dc58 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/d7cd5c9cb2f84d029eab38952237dc58 2024-12-02T22:16:11,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/802bfc315bd144bcab44b28d31f850e2 2024-12-02T22:16:11,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/e1840087ef054a88984f2012151527e1 2024-12-02T22:16:11,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/374e2041b60c4bc78529cab50dd18cbe 2024-12-02T22:16:11,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/0ad4a63d3bdf4967bd99bf608f5b8cf6 2024-12-02T22:16:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/484c3ddd65d24ce49ed1e002d1069c90 2024-12-02T22:16:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/db4778c205504694a6a033d3ec85416b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/db4778c205504694a6a033d3ec85416b 2024-12-02T22:16:11,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2e21aec478be4dbf9e3bb5f882ccf5e9 2024-12-02T22:16:11,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ea91c0215ae64c82a8305bba9c379cd4 2024-12-02T22:16:11,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a61747b8203e45e2951252d0c047ea72 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/a61747b8203e45e2951252d0c047ea72 2024-12-02T22:16:11,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/4e4df3eb6cad40249d5ce50331990347 2024-12-02T22:16:11,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/2f752b67a4a84f5fad96f1444170eafa 2024-12-02T22:16:11,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b3793c5b84c848b29e8b2eea3b4c8340 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/b3793c5b84c848b29e8b2eea3b4c8340 2024-12-02T22:16:11,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/19e820c270f941929788d042bc227f05 2024-12-02T22:16:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/90cd658511584ea09f4cacf5c7e6fde5 2024-12-02T22:16:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ffa2f98a7bad4ef78b082b18657b02af 2024-12-02T22:16:11,631 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/recovered.edits/568.seqid, newMaxSeqId=568, maxSeqId=1 2024-12-02T22:16:11,632 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. 2024-12-02T22:16:11,632 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 1fbeb7002d31ceedab10535986bc0bcf: Waiting for close lock at 1733177769235Running coprocessor pre-close hooks at 1733177769235Disabling compacts and flushes for region at 1733177769235Disabling writes for close at 1733177770303 (+1068 ms)Obtaining lock to block concurrent updates at 1733177770303Preparing flush snapshotting stores in 1fbeb7002d31ceedab10535986bc0bcf at 1733177770303Finished memstore snapshotting TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1733177770304 (+1 ms)Flushing stores of TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf. at 1733177770304Flushing 1fbeb7002d31ceedab10535986bc0bcf/A: creating writer at 1733177770304Flushing 1fbeb7002d31ceedab10535986bc0bcf/A: appending metadata at 1733177770306 (+2 ms)Flushing 1fbeb7002d31ceedab10535986bc0bcf/A: closing flushed file at 1733177770306Flushing 1fbeb7002d31ceedab10535986bc0bcf/B: creating writer at 1733177770714 (+408 ms)Flushing 1fbeb7002d31ceedab10535986bc0bcf/B: appending metadata at 1733177770716 (+2 ms)Flushing 1fbeb7002d31ceedab10535986bc0bcf/B: closing flushed file at 1733177770716Flushing 1fbeb7002d31ceedab10535986bc0bcf/C: creating writer at 1733177771125 (+409 ms)Flushing 1fbeb7002d31ceedab10535986bc0bcf/C: appending metadata at 1733177771127 (+2 ms)Flushing 1fbeb7002d31ceedab10535986bc0bcf/C: closing flushed file at 1733177771127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ef08ebd: reopening flushed file at 1733177771534 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c62c49: reopening flushed file at 1733177771537 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41e07e82: reopening flushed file at 1733177771541 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 1fbeb7002d31ceedab10535986bc0bcf in 1241ms, sequenceid=565, compaction requested=true at 1733177771544 (+3 ms)Writing region close event to WAL at 1733177771629 (+85 ms)Running coprocessor post-close hooks at 1733177771632 (+3 ms)Closed at 1733177771632 2024-12-02T22:16:11,634 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:11,634 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1fbeb7002d31ceedab10535986bc0bcf, regionState=CLOSED 2024-12-02T22:16:11,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:16:11,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-02T22:16:11,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fbeb7002d31ceedab10535986bc0bcf, server=7ae73638d47e,41753,1733177627056 in 2.5530 sec 2024-12-02T22:16:11,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-02T22:16:11,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fbeb7002d31ceedab10535986bc0bcf, UNASSIGN in 2.5560 sec 2024-12-02T22:16:11,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-02T22:16:11,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5590 sec 2024-12-02T22:16:11,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177771641"}]},"ts":"1733177771641"} 2024-12-02T22:16:11,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:16:11,642 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:16:11,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5670 sec 2024-12-02T22:16:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T22:16:13,233 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:13,233 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:16:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,235 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T22:16:13,236 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:13,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/recovered.edits] 2024-12-02T22:16:13,242 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/26169fba43ad4bae8c21fe63883b9f88 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/26169fba43ad4bae8c21fe63883b9f88 2024-12-02T22:16:13,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/456ce995365f4872b01a0ea6106fc075 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/456ce995365f4872b01a0ea6106fc075 2024-12-02T22:16:13,244 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/966d82e9165b450896db2af6f34443b6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/966d82e9165b450896db2af6f34443b6 2024-12-02T22:16:13,245 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/ae1568e4d8a448feba675cc968b2592b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/A/ae1568e4d8a448feba675cc968b2592b 2024-12-02T22:16:13,247 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90009348ce9a4be9afbf8185beab89ef to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90009348ce9a4be9afbf8185beab89ef 2024-12-02T22:16:13,248 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90570c0b5dcc4742b6f430f9e0dc284c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/90570c0b5dcc4742b6f430f9e0dc284c 2024-12-02T22:16:13,249 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b671b917598b4ab795e810986d2579d1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/b671b917598b4ab795e810986d2579d1 2024-12-02T22:16:13,250 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ba42a2478c0d4831a8e76c0c13a87ac8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/B/ba42a2478c0d4831a8e76c0c13a87ac8 2024-12-02T22:16:13,252 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/15624212d62f43df8db3fb3b0163f271 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/15624212d62f43df8db3fb3b0163f271 2024-12-02T22:16:13,253 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1b85a8211d5543c99060374ad2af6063 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/1b85a8211d5543c99060374ad2af6063 2024-12-02T22:16:13,254 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/794cdc8e310345e4acc968f36b53641b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/794cdc8e310345e4acc968f36b53641b 2024-12-02T22:16:13,254 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ebf17e2eeab3428cacd442511ad16d0e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/C/ebf17e2eeab3428cacd442511ad16d0e 2024-12-02T22:16:13,256 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/recovered.edits/568.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf/recovered.edits/568.seqid 2024-12-02T22:16:13,257 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/1fbeb7002d31ceedab10535986bc0bcf 2024-12-02T22:16:13,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:16:13,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,261 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:16:13,262 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:16:13,263 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,263 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:16:13,264 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177773263"}]},"ts":"9223372036854775807"} 2024-12-02T22:16:13,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:16:13,265 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1fbeb7002d31ceedab10535986bc0bcf, NAME => 'TestAcidGuarantees,,1733177744173.1fbeb7002d31ceedab10535986bc0bcf.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:16:13,265 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:16:13,265 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177773265"}]},"ts":"9223372036854775807"} 2024-12-02T22:16:13,267 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:16:13,267 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-12-02T22:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T22:16:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:16:13,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:13,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 248), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=471 (was 564), ProcessCount=11 (was 11), AvailableMemoryMB=7215 (was 7293) 2024-12-02T22:16:13,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=471, ProcessCount=11, AvailableMemoryMB=7215 2024-12-02T22:16:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:16:13,362 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:13,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T22:16:13,364 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:13,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-12-02T22:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:13,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T22:16:13,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742417_1593 (size=1061) 2024-12-02T22:16:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:13,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c 2024-12-02T22:16:13,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742418_1594 (size=53) 2024-12-02T22:16:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a1bc7c7f9af7b8cbed3248a7876b3899, disabling compactions & flushes 2024-12-02T22:16:14,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. after waiting 0 ms 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1bc7c7f9af7b8cbed3248a7876b3899: Waiting for close lock at 1733177774176Disabling compacts and flushes for region at 1733177774176Disabling writes for close at 1733177774176Writing region close event to WAL at 1733177774176Closed at 1733177774176 2024-12-02T22:16:14,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T22:16:14,177 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733177774177"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733177774177"}]},"ts":"1733177774177"} 2024-12-02T22:16:14,178 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T22:16:14,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T22:16:14,179 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177774179"}]},"ts":"1733177774179"} 2024-12-02T22:16:14,180 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T22:16:14,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, ASSIGN}] 2024-12-02T22:16:14,181 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, ASSIGN 2024-12-02T22:16:14,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, ASSIGN; state=OFFLINE, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=false 2024-12-02T22:16:14,332 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:14,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, ASSIGN because future has completed 2024-12-02T22:16:14,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:16:14,487 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,487 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:16:14,487 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,487 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:16:14,487 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,487 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,488 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,489 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:14,489 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName A 2024-12-02T22:16:14,490 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:14,490 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:14,490 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,491 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:14,491 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName B 2024-12-02T22:16:14,491 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:14,491 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:14,491 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,492 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:14,492 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName C 2024-12-02T22:16:14,492 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:14,492 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:14,492 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,492 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,493 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,493 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,494 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,494 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,494 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:16:14,495 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,497 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T22:16:14,497 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened a1bc7c7f9af7b8cbed3248a7876b3899; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70995153, jitterRate=0.057910218834877014}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:16:14,497 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:14,498 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for a1bc7c7f9af7b8cbed3248a7876b3899: Running coprocessor pre-open hook at 1733177774488Writing region info on filesystem at 1733177774488Initializing all the Stores at 1733177774488Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177774488Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177774488Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177774488Cleaning up temporary data from old regions at 1733177774494 (+6 ms)Running coprocessor post-open hooks at 1733177774497 (+3 ms)Region opened successfully at 1733177774498 (+1 ms) 2024-12-02T22:16:14,498 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., pid=152, masterSystemTime=1733177774485 2024-12-02T22:16:14,500 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,500 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:14,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=OPEN, openSeqNum=2, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:14,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:16:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:14,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-02T22:16:14,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 in 168 msec 2024-12-02T22:16:14,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-02T22:16:14,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, ASSIGN in 323 msec 2024-12-02T22:16:14,506 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T22:16:14,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177774506"}]},"ts":"1733177774506"} 2024-12-02T22:16:14,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T22:16:14,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T22:16:14,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-12-02T22:16:15,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T22:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T22:16:15,513 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:15,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec06c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:15,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:15,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:15,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582164a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:15,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bec64f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:15,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:15,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:15,519 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T22:16:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7ae73638d47e,36511,1733177626309 2024-12-02T22:16:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@428404d2 2024-12-02T22:16:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T22:16:15,520 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T22:16:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T22:16:15,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T22:16:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:15,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742419_1595 (size=1097) 2024-12-02T22:16:15,932 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T22:16:15,932 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T22:16:15,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:16:15,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE}] 2024-12-02T22:16:15,936 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE 2024-12-02T22:16:15,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:15,938 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36511 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=7ae73638d47e,41753,1733177627056, table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T22:16:15,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE because future has completed 2024-12-02T22:16:15,938 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:16:15,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:16:16,090 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,090 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:16:16,090 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing a1bc7c7f9af7b8cbed3248a7876b3899, disabling compactions & flushes 2024-12-02T22:16:16,090 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,090 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,090 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. after waiting 0 ms 2024-12-02T22:16:16,090 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,093 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T22:16:16,094 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,094 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for a1bc7c7f9af7b8cbed3248a7876b3899: Waiting for close lock at 1733177776090Running coprocessor pre-close hooks at 1733177776090Disabling compacts and flushes for region at 1733177776090Disabling writes for close at 1733177776090Writing region close event to WAL at 1733177776091 (+1 ms)Running coprocessor post-close hooks at 1733177776094 (+3 ms)Closed at 1733177776094 2024-12-02T22:16:16,094 WARN [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: a1bc7c7f9af7b8cbed3248a7876b3899 to self. 2024-12-02T22:16:16,095 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,096 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=CLOSED 2024-12-02T22:16:16,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:16:16,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-12-02T22:16:16,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 in 159 msec 2024-12-02T22:16:16,099 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE; state=CLOSED, location=7ae73638d47e,41753,1733177627056; forceNewPlan=false, retain=true 2024-12-02T22:16:16,250 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=OPENING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE because future has completed 2024-12-02T22:16:16,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:16:16,405 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,405 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} 2024-12-02T22:16:16,405 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,405 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T22:16:16,406 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,406 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,407 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,407 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:16,407 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName A 2024-12-02T22:16:16,408 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:16,408 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:16,409 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,409 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:16,410 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName B 2024-12-02T22:16:16,410 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:16,410 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:16,410 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,411 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T22:16:16,411 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1bc7c7f9af7b8cbed3248a7876b3899 columnFamilyName C 2024-12-02T22:16:16,411 DEBUG [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:16,411 INFO [StoreOpener-a1bc7c7f9af7b8cbed3248a7876b3899-1 {}] regionserver.HStore(327): Store=a1bc7c7f9af7b8cbed3248a7876b3899/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T22:16:16,411 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,411 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,412 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,413 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,413 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,413 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,414 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T22:16:16,414 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,415 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened a1bc7c7f9af7b8cbed3248a7876b3899; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59125804, jitterRate=-0.11895686388015747}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T22:16:16,415 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,416 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for a1bc7c7f9af7b8cbed3248a7876b3899: Running coprocessor pre-open hook at 1733177776406Writing region info on filesystem at 1733177776406Initializing all the Stores at 1733177776406Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177776406Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177776406Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733177776406Cleaning up temporary data from old regions at 1733177776413 (+7 ms)Running coprocessor post-open hooks at 1733177776415 (+2 ms)Region opened successfully at 1733177776415 2024-12-02T22:16:16,416 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., pid=157, masterSystemTime=1733177776403 2024-12-02T22:16:16,418 DEBUG [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,418 INFO [RS_OPEN_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=OPEN, openSeqNum=5, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:16:16,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-12-02T22:16:16,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 in 169 msec 2024-12-02T22:16:16,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T22:16:16,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, REOPEN/MOVE in 486 msec 2024-12-02T22:16:16,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-02T22:16:16,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-12-02T22:16:16,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-12-02T22:16:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T22:16:16,428 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:16,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b84fc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d00b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,434 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bad1fdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509650bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fa6301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df4b973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395e4509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,441 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb8d07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67532e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,444 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bd86ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bd29e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251c8d7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d0e7f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7761e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,451 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8e6c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,452 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e843f07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc9844f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7ae73638d47e,36511,-1 for getting cluster id 2024-12-02T22:16:16,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T22:16:16,455 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ffa625a2-8222-4ade-8d4e-7a91ac6a7448' 2024-12-02T22:16:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T22:16:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ffa625a2-8222-4ade-8d4e-7a91ac6a7448" 2024-12-02T22:16:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64259f49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7ae73638d47e,36511,-1] 2024-12-02T22:16:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T22:16:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:16,456 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T22:16:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82378bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T22:16:16,459 DEBUG [Thread-2622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,459 DEBUG [Thread-2630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,459 DEBUG [Thread-2624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,460 DEBUG [Thread-2633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,460 DEBUG [Thread-2626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,462 DEBUG [Thread-2628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,463 DEBUG [Thread-2639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,464 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:16,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-02T22:16:16,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,467 DEBUG [Thread-2635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,467 DEBUG [Thread-2637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,467 DEBUG [Thread-2641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T22:16:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:16,467 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:16,468 DEBUG [Thread-2641 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,468 DEBUG [Thread-2641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7ae73638d47e,41753,1733177627056, seqNum=-1] 2024-12-02T22:16:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T22:16:16,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,474 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:16,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T22:16:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,477 DEBUG [Thread-2641 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5] 2024-12-02T22:16:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:16,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bfc2f964aa52482ab2c868fdca71386f_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177776477/Put/seqid=0 2024-12-02T22:16:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177836502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177836502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177776497, max timestamp=1733177776507Num uniques 859; 2024-12-02T22:16:16,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:16:16,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177776497, max timestamp=1733177776507Num uniques 859; 2024-12-02T22:16:16,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:16:16,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733177776497, max timestamp=1733177776507Num uniques 859; 2024-12-02T22:16:16,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T22:16:16,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742420_1596 (size=12154) 2024-12-02T22:16:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177836536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177836537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177836538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177836613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177836613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177836643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177836654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177836654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:16,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177836823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177836823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177836853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177836862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177836863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:16,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:16,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bfc2f964aa52482ab2c868fdca71386f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bfc2f964aa52482ab2c868fdca71386f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:16,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/3475b951c16649b6b96c8f30999dacc8, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:16,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/3475b951c16649b6b96c8f30999dacc8 is 175, key is test_row_0/A:col10/1733177776477/Put/seqid=0 2024-12-02T22:16:16,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742421_1597 (size=30955) 2024-12-02T22:16:16,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/3475b951c16649b6b96c8f30999dacc8 2024-12-02T22:16:16,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:16,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:16,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/8dbde260e3f64e41ad5faf9497b12776 is 50, key is test_row_0/B:col10/1733177776477/Put/seqid=0 2024-12-02T22:16:16,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742422_1598 (size=12001) 2024-12-02T22:16:16,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/8dbde260e3f64e41ad5faf9497b12776 2024-12-02T22:16:16,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f20ef912685b4d009f04775f313789b9 is 50, key is test_row_0/C:col10/1733177776477/Put/seqid=0 2024-12-02T22:16:16,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742423_1599 (size=12001) 2024-12-02T22:16:17,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:17,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177837133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177837133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,153 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:16:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177837163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177837173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177837174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f20ef912685b4d009f04775f313789b9 2024-12-02T22:16:17,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/3475b951c16649b6b96c8f30999dacc8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8 2024-12-02T22:16:17,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T22:16:17,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/8dbde260e3f64e41ad5faf9497b12776 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776 2024-12-02T22:16:17,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T22:16:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f20ef912685b4d009f04775f313789b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9 2024-12-02T22:16:17,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T22:16:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a1bc7c7f9af7b8cbed3248a7876b3899 in 921ms, sequenceid=15, compaction requested=false 2024-12-02T22:16:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T22:16:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:17,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T22:16:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:17,545 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120204b788909035427dbbebe3ccd851fa95_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177776534/Put/seqid=0 2024-12-02T22:16:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742424_1600 (size=12154) 2024-12-02T22:16:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177837661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177837664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177837672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177837683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177837683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177837773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177837773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:17,963 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120204b788909035427dbbebe3ccd851fa95_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120204b788909035427dbbebe3ccd851fa95_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a230bfc0fa7e455b906752fe8fa93b01, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a230bfc0fa7e455b906752fe8fa93b01 is 175, key is test_row_0/A:col10/1733177776534/Put/seqid=0 2024-12-02T22:16:17,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742425_1601 (size=30955) 2024-12-02T22:16:17,974 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a230bfc0fa7e455b906752fe8fa93b01 2024-12-02T22:16:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177837983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177837983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/386f35a0bdf5482fa983fbd139e502d8 is 50, key is test_row_0/B:col10/1733177776534/Put/seqid=0 2024-12-02T22:16:17,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742426_1602 (size=12001) 2024-12-02T22:16:17,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/386f35a0bdf5482fa983fbd139e502d8 2024-12-02T22:16:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/04a44e752e104031b38dab5d6f033bb8 is 50, key is test_row_0/C:col10/1733177776534/Put/seqid=0 2024-12-02T22:16:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742427_1603 (size=12001) 2024-12-02T22:16:18,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/04a44e752e104031b38dab5d6f033bb8 2024-12-02T22:16:18,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a230bfc0fa7e455b906752fe8fa93b01 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01 2024-12-02T22:16:18,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01, entries=150, sequenceid=50, filesize=30.2 K 2024-12-02T22:16:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/386f35a0bdf5482fa983fbd139e502d8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8 2024-12-02T22:16:18,025 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T22:16:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/04a44e752e104031b38dab5d6f033bb8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8 2024-12-02T22:16:18,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T22:16:18,030 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for a1bc7c7f9af7b8cbed3248a7876b3899 in 486ms, sequenceid=50, compaction requested=false 2024-12-02T22:16:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-02T22:16:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-02T22:16:18,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T22:16:18,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-12-02T22:16:18,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5720 sec 2024-12-02T22:16:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:18,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028b447ff15a7e46459c1b4d52a1a596ba_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742428_1604 (size=12154) 2024-12-02T22:16:18,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:18,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028b447ff15a7e46459c1b4d52a1a596ba_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028b447ff15a7e46459c1b4d52a1a596ba_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:18,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/720e477cc88e40788e95c7a507b3cda8, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:18,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/720e477cc88e40788e95c7a507b3cda8 is 175, key is test_row_0/A:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742429_1605 (size=30955) 2024-12-02T22:16:18,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/720e477cc88e40788e95c7a507b3cda8 2024-12-02T22:16:18,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5b613377206a4780aa6bfc6674f66d14 is 50, key is test_row_0/B:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742430_1606 (size=12001) 2024-12-02T22:16:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177838338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177838341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177838453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177838453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T22:16:18,624 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:18,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-02T22:16:18,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:18,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:18,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:18,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177838663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177838663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177838684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,688 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177838693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,696 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177838703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,708 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_0' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:18,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5b613377206a4780aa6bfc6674f66d14 2024-12-02T22:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:18,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/74772448204140e4a90af9fa03365492 is 50, key is test_row_0/C:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742431_1607 (size=12001) 2024-12-02T22:16:18,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/74772448204140e4a90af9fa03365492 2024-12-02T22:16:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/720e477cc88e40788e95c7a507b3cda8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8 2024-12-02T22:16:18,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8, entries=150, sequenceid=62, filesize=30.2 K 2024-12-02T22:16:18,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5b613377206a4780aa6bfc6674f66d14 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14 2024-12-02T22:16:18,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T22:16:18,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/74772448204140e4a90af9fa03365492 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492 2024-12-02T22:16:18,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T22:16:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a1bc7c7f9af7b8cbed3248a7876b3899 in 472ms, sequenceid=62, compaction requested=true 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:18,767 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:18,767 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:18,769 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:18,769 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:18,770 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:18,770 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:18,770 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=90.7 K 2024-12-02T22:16:18,770 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=35.2 K 2024-12-02T22:16:18,770 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8] 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbde260e3f64e41ad5faf9497b12776, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177776465 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3475b951c16649b6b96c8f30999dacc8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177776465 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 386f35a0bdf5482fa983fbd139e502d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733177776534 2024-12-02T22:16:18,770 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a230bfc0fa7e455b906752fe8fa93b01, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733177776534 2024-12-02T22:16:18,771 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b613377206a4780aa6bfc6674f66d14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:18,771 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720e477cc88e40788e95c7a507b3cda8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:18,783 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:18,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:18,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:18,790 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:18,791 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/c434b2cce16b4bd499471d49bf2be7cf is 50, key is test_row_0/B:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,797 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202f621f1b6344648e49a6506cdb484bc35_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:18,799 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202f621f1b6344648e49a6506cdb484bc35_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:18,799 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f621f1b6344648e49a6506cdb484bc35_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:18,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273651747ee8a4785a8c3beaa76ded420_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177778339/Put/seqid=0 2024-12-02T22:16:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742432_1608 (size=12104) 2024-12-02T22:16:18,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742433_1609 (size=4469) 2024-12-02T22:16:18,879 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#522 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:18,879 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6f92d65a1dfb4aada371ebba98b5d3e2 is 175, key is test_row_0/A:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742434_1610 (size=12154) 2024-12-02T22:16:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742435_1611 (size=31058) 2024-12-02T22:16:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:18,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177838994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177838998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177839103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177839113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:19,267 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/c434b2cce16b4bd499471d49bf2be7cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c434b2cce16b4bd499471d49bf2be7cf 2024-12-02T22:16:19,271 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into c434b2cce16b4bd499471d49bf2be7cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:19,271 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:19,271 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177778767; duration=0sec 2024-12-02T22:16:19,271 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:19,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:19,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:19,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:19,272 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:19,272 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:19,272 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=35.2 K 2024-12-02T22:16:19,273 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f20ef912685b4d009f04775f313789b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733177776465 2024-12-02T22:16:19,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a44e752e104031b38dab5d6f033bb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733177776534 2024-12-02T22:16:19,274 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 74772448204140e4a90af9fa03365492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:19,282 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#524 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:19,282 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c0d9f19fa0b4f188859868463917560 is 50, key is test_row_0/C:col10/1733177777653/Put/seqid=0 2024-12-02T22:16:19,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742436_1612 (size=12104) 2024-12-02T22:16:19,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:19,305 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6f92d65a1dfb4aada371ebba98b5d3e2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2 2024-12-02T22:16:19,307 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273651747ee8a4785a8c3beaa76ded420_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273651747ee8a4785a8c3beaa76ded420_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:19,307 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c0d9f19fa0b4f188859868463917560 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c0d9f19fa0b4f188859868463917560 2024-12-02T22:16:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6519417f3ee640cea8e04bfee7bcd0bf, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:19,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6519417f3ee640cea8e04bfee7bcd0bf is 175, key is test_row_0/A:col10/1733177778339/Put/seqid=0 2024-12-02T22:16:19,312 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 6f92d65a1dfb4aada371ebba98b5d3e2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:19,312 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:19,313 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177778767; duration=0sec 2024-12-02T22:16:19,313 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:19,313 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:19,316 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 1c0d9f19fa0b4f188859868463917560(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:19,316 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:19,316 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=13, startTime=1733177778767; duration=0sec 2024-12-02T22:16:19,316 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:19,316 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742437_1613 (size=30955) 2024-12-02T22:16:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177839313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177839323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177839633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177839633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:19,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6519417f3ee640cea8e04bfee7bcd0bf 2024-12-02T22:16:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/c3242c19c2764569a1d9514cef400afb is 50, key is test_row_0/B:col10/1733177778339/Put/seqid=0 2024-12-02T22:16:19,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742438_1614 (size=12001) 2024-12-02T22:16:19,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/c3242c19c2764569a1d9514cef400afb 2024-12-02T22:16:19,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c7006de0af34ec4904f4ed29f122d7c is 50, key is test_row_0/C:col10/1733177778339/Put/seqid=0 2024-12-02T22:16:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742439_1615 (size=12001) 2024-12-02T22:16:19,743 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c7006de0af34ec4904f4ed29f122d7c 2024-12-02T22:16:19,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/6519417f3ee640cea8e04bfee7bcd0bf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf 2024-12-02T22:16:19,751 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf, entries=150, sequenceid=82, filesize=30.2 K 2024-12-02T22:16:19,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/c3242c19c2764569a1d9514cef400afb as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb 2024-12-02T22:16:19,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T22:16:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c7006de0af34ec4904f4ed29f122d7c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c 2024-12-02T22:16:19,762 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T22:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:19,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a1bc7c7f9af7b8cbed3248a7876b3899 in 979ms, sequenceid=82, compaction requested=false 2024-12-02T22:16:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:19,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-02T22:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-02T22:16:19,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-02T22:16:19,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1390 sec 2024-12-02T22:16:19,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1450 sec 2024-12-02T22:16:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:20,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:20,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202694e5829087743e7882898a437b17486_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742440_1616 (size=17034) 2024-12-02T22:16:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177840187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177840190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177840303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177840303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177840513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177840514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,562 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:20,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202694e5829087743e7882898a437b17486_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202694e5829087743e7882898a437b17486_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:20,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a26eb72f1c5c4a12ba0c35a6919a3036, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:20,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a26eb72f1c5c4a12ba0c35a6919a3036 is 175, key is test_row_0/A:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742441_1617 (size=48139) 2024-12-02T22:16:20,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a26eb72f1c5c4a12ba0c35a6919a3036 2024-12-02T22:16:20,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bb2920b964cc4a92abc3c78368978ebc is 50, key is test_row_0/B:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742442_1618 (size=12001) 2024-12-02T22:16:20,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bb2920b964cc4a92abc3c78368978ebc 2024-12-02T22:16:20,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e5b9edd6ecf34c87a708cf6de82d4058 is 50, key is test_row_0/C:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742443_1619 (size=12001) 2024-12-02T22:16:20,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e5b9edd6ecf34c87a708cf6de82d4058 2024-12-02T22:16:20,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a26eb72f1c5c4a12ba0c35a6919a3036 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036 2024-12-02T22:16:20,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036, entries=250, sequenceid=99, filesize=47.0 K 2024-12-02T22:16:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bb2920b964cc4a92abc3c78368978ebc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc 2024-12-02T22:16:20,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc, entries=150, sequenceid=99, filesize=11.7 K 2024-12-02T22:16:20,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e5b9edd6ecf34c87a708cf6de82d4058 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058 2024-12-02T22:16:20,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058, entries=150, sequenceid=99, filesize=11.7 K 2024-12-02T22:16:20,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a1bc7c7f9af7b8cbed3248a7876b3899 in 479ms, sequenceid=99, compaction requested=true 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:20,626 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:20,626 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:20,627 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:20,627 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:20,627 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,627 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c434b2cce16b4bd499471d49bf2be7cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=35.3 K 2024-12-02T22:16:20,627 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:20,627 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:20,627 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,627 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=107.6 K 2024-12-02T22:16:20,627 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,627 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036] 2024-12-02T22:16:20,629 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c434b2cce16b4bd499471d49bf2be7cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:20,629 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f92d65a1dfb4aada371ebba98b5d3e2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:20,630 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3242c19c2764569a1d9514cef400afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733177778304 2024-12-02T22:16:20,630 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6519417f3ee640cea8e04bfee7bcd0bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733177778304 2024-12-02T22:16:20,631 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2920b964cc4a92abc3c78368978ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:20,631 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a26eb72f1c5c4a12ba0c35a6919a3036, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:20,641 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:20,643 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:20,643 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/3ce292a29a554e72aa3a01c80af6baee is 50, key is test_row_0/B:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,649 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202202e77c0a97641c49c933ad80b6ea402_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:20,651 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202202e77c0a97641c49c933ad80b6ea402_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:20,651 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202202e77c0a97641c49c933ad80b6ea402_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:20,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742444_1620 (size=12207) 2024-12-02T22:16:20,685 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/3ce292a29a554e72aa3a01c80af6baee as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/3ce292a29a554e72aa3a01c80af6baee 2024-12-02T22:16:20,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742445_1621 (size=4469) 2024-12-02T22:16:20,693 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 3ce292a29a554e72aa3a01c80af6baee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:20,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:20,693 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177780626; duration=0sec 2024-12-02T22:16:20,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:20,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:20,693 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:20,695 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:20,695 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:20,695 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,695 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c0d9f19fa0b4f188859868463917560, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=35.3 K 2024-12-02T22:16:20,696 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0d9f19fa0b4f188859868463917560, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733177777653 2024-12-02T22:16:20,697 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7006de0af34ec4904f4ed29f122d7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733177778304 2024-12-02T22:16:20,697 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b9edd6ecf34c87a708cf6de82d4058, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:20,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:16:20,705 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:20,705 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9b39dcaf37dd41198d80b5956695cbf0 is 50, key is test_row_0/C:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:20,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c0de651d68ef424aa6ec482ab412d54e_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177780182/Put/seqid=0 2024-12-02T22:16:20,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742446_1622 (size=12207) 2024-12-02T22:16:20,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742447_1623 (size=14594) 2024-12-02T22:16:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177840739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177840740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177840743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T22:16:20,784 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:20,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-02T22:16:20,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:20,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:20,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:20,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177840853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177840853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177840853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:20,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:20,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:20,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177841063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177841063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177841063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,091 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#530 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:21,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,092 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d is 175, key is test_row_0/A:col10/1733177780145/Put/seqid=0 2024-12-02T22:16:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742448_1624 (size=31161) 2024-12-02T22:16:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:21,133 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9b39dcaf37dd41198d80b5956695cbf0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9b39dcaf37dd41198d80b5956695cbf0 2024-12-02T22:16:21,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:21,139 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 9b39dcaf37dd41198d80b5956695cbf0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:21,139 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:21,139 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=13, startTime=1733177780626; duration=0sec 2024-12-02T22:16:21,139 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:21,139 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:21,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c0de651d68ef424aa6ec482ab412d54e_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0de651d68ef424aa6ec482ab412d54e_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:21,142 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f226078312a24aa2981ff18d29c71791, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:21,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f226078312a24aa2981ff18d29c71791 is 175, key is test_row_0/A:col10/1733177780182/Put/seqid=0 2024-12-02T22:16:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742449_1625 (size=39549) 2024-12-02T22:16:21,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177841333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177841333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177841373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177841373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177841373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:21,501 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d 2024-12-02T22:16:21,506 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into e4aa3ec13f3d4d7f9d622b7aa5e61a0d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:21,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:21,506 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177780626; duration=0sec 2024-12-02T22:16:21,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:21,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:21,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f226078312a24aa2981ff18d29c71791 2024-12-02T22:16:21,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/e4c1c97a2bfe452fae1934482597c66c is 50, key is test_row_0/B:col10/1733177780182/Put/seqid=0 2024-12-02T22:16:21,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742450_1626 (size=12001) 2024-12-02T22:16:21,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:21,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:21,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177841883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177841893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177841893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:21,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/e4c1c97a2bfe452fae1934482597c66c 2024-12-02T22:16:21,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c771c6aac4544c1b4f3e42fe57c78f4 is 50, key is test_row_0/C:col10/1733177780182/Put/seqid=0 2024-12-02T22:16:21,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742451_1627 (size=12001) 2024-12-02T22:16:22,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177842353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177842353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:22,355 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:22,356 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:22,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c771c6aac4544c1b4f3e42fe57c78f4 2024-12-02T22:16:22,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f226078312a24aa2981ff18d29c71791 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791 2024-12-02T22:16:22,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791, entries=200, sequenceid=119, filesize=38.6 K 2024-12-02T22:16:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/e4c1c97a2bfe452fae1934482597c66c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c 2024-12-02T22:16:22,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T22:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1c771c6aac4544c1b4f3e42fe57c78f4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4 2024-12-02T22:16:22,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T22:16:22,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1682ms, sequenceid=119, compaction requested=false 2024-12-02T22:16:22,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:22,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:22,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:22,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202af2e3294ad4e4e69bf50bddc62ca381d_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177780742/Put/seqid=0 2024-12-02T22:16:22,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742452_1628 (size=12254) 2024-12-02T22:16:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:22,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202af2e3294ad4e4e69bf50bddc62ca381d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202af2e3294ad4e4e69bf50bddc62ca381d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/93749ea02c9741f6a161301c50abbc4f, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:22,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/93749ea02c9741f6a161301c50abbc4f is 175, key is test_row_0/A:col10/1733177780742/Put/seqid=0 2024-12-02T22:16:22,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742453_1629 (size=31055) 2024-12-02T22:16:22,888 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/93749ea02c9741f6a161301c50abbc4f 2024-12-02T22:16:22,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/378203cb233347958e7a6da754bb6ad4 is 50, key is test_row_0/B:col10/1733177780742/Put/seqid=0 2024-12-02T22:16:22,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742454_1630 (size=12101) 2024-12-02T22:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:22,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177842931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177842934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177842936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177843043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177843043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177843043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177843253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177843253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177843253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/378203cb233347958e7a6da754bb6ad4 2024-12-02T22:16:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/21010efd874c43a9b2633623282d3481 is 50, key is test_row_0/C:col10/1733177780742/Put/seqid=0 2024-12-02T22:16:23,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742455_1631 (size=12101) 2024-12-02T22:16:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177843564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177843564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177843564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:23,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/21010efd874c43a9b2633623282d3481 2024-12-02T22:16:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/93749ea02c9741f6a161301c50abbc4f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f 2024-12-02T22:16:23,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f, entries=150, sequenceid=134, filesize=30.3 K 2024-12-02T22:16:23,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/378203cb233347958e7a6da754bb6ad4 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4 2024-12-02T22:16:23,724 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4, entries=150, sequenceid=134, filesize=11.8 K 2024-12-02T22:16:23,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/21010efd874c43a9b2633623282d3481 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481 2024-12-02T22:16:23,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481, entries=150, sequenceid=134, filesize=11.8 K 2024-12-02T22:16:23,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1260ms, sequenceid=134, compaction requested=true 2024-12-02T22:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-02T22:16:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-02T22:16:23,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T22:16:23,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9460 sec 2024-12-02T22:16:23,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.9510 sec 2024-12-02T22:16:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202192330cd998b47f1981106e2250b83d4_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742456_1632 (size=14794) 2024-12-02T22:16:24,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:24,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202192330cd998b47f1981106e2250b83d4_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202192330cd998b47f1981106e2250b83d4_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,093 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ab1e087faa7d41de80b2cbcad70f05bf, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ab1e087faa7d41de80b2cbcad70f05bf is 175, key is test_row_0/A:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742457_1633 (size=39749) 2024-12-02T22:16:24,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177844094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177844100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177844101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177844203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177844213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177844213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177844363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,366 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_2' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177844383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,389 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7ae73638d47e,41753,1733177627056 for 'test_row_1' in a1bc7c7f9af7b8cbed3248a7876b3899 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T22:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177844413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177844423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177844423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ab1e087faa7d41de80b2cbcad70f05bf 2024-12-02T22:16:24,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/fc63d6bbe30743e99d1151a9f914c82e is 50, key is test_row_0/B:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742458_1634 (size=12151) 2024-12-02T22:16:24,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/fc63d6bbe30743e99d1151a9f914c82e 2024-12-02T22:16:24,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1e7d994b14f8403ca77bf70194d15a01 is 50, key is test_row_0/C:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742459_1635 (size=12151) 2024-12-02T22:16:24,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1e7d994b14f8403ca77bf70194d15a01 2024-12-02T22:16:24,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ab1e087faa7d41de80b2cbcad70f05bf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf 2024-12-02T22:16:24,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf, entries=200, sequenceid=155, filesize=38.8 K 2024-12-02T22:16:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/fc63d6bbe30743e99d1151a9f914c82e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e 2024-12-02T22:16:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T22:16:24,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/1e7d994b14f8403ca77bf70194d15a01 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01 2024-12-02T22:16:24,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T22:16:24,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for a1bc7c7f9af7b8cbed3248a7876b3899 in 466ms, sequenceid=155, compaction requested=true 2024-12-02T22:16:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:24,542 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:24,543 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:24,543 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:24,543 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:24,543 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=138.2 K 2024-12-02T22:16:24,543 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:24,543 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf] 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:24,544 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4aa3ec13f3d4d7f9d622b7aa5e61a0d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,544 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:24,545 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f226078312a24aa2981ff18d29c71791, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733177780182 2024-12-02T22:16:24,545 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93749ea02c9741f6a161301c50abbc4f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733177780728 2024-12-02T22:16:24,545 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab1e087faa7d41de80b2cbcad70f05bf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733177782926 2024-12-02T22:16:24,545 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:24,546 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:24,546 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:24,547 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/3ce292a29a554e72aa3a01c80af6baee, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=47.3 K 2024-12-02T22:16:24,548 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce292a29a554e72aa3a01c80af6baee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:24,548 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c1c97a2bfe452fae1934482597c66c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733177780182 2024-12-02T22:16:24,549 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 378203cb233347958e7a6da754bb6ad4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733177780728 2024-12-02T22:16:24,549 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting fc63d6bbe30743e99d1151a9f914c82e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733177782926 2024-12-02T22:16:24,553 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,555 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412024362ca7b5fd747cf8bab59ebc7ea0690_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,559 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412024362ca7b5fd747cf8bab59ebc7ea0690_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,561 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024362ca7b5fd747cf8bab59ebc7ea0690_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,567 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#543 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:24,568 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/a768d70ce1a14b4da0a6da4ca0970d1f is 50, key is test_row_0/B:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742460_1636 (size=12493) 2024-12-02T22:16:24,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742461_1637 (size=4469) 2024-12-02T22:16:24,606 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#542 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:24,607 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/36a36d7965794549aa1dc15993463b3c is 175, key is test_row_0/A:col10/1733177782930/Put/seqid=0 2024-12-02T22:16:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742462_1638 (size=31447) 2024-12-02T22:16:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:24,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202383ac636fbde44a9a9363fee8d92cb3d_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177784724/Put/seqid=0 2024-12-02T22:16:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742463_1639 (size=14794) 2024-12-02T22:16:24,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:24,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202383ac636fbde44a9a9363fee8d92cb3d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202383ac636fbde44a9a9363fee8d92cb3d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/b097e72c934146da91d4a6839bdc92a7, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/b097e72c934146da91d4a6839bdc92a7 is 175, key is test_row_0/A:col10/1733177784724/Put/seqid=0 2024-12-02T22:16:24,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742464_1640 (size=39749) 2024-12-02T22:16:24,751 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/b097e72c934146da91d4a6839bdc92a7 2024-12-02T22:16:24,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/6d8d23e800fe488ea47bd1093aab2a87 is 50, key is test_row_0/B:col10/1733177784724/Put/seqid=0 2024-12-02T22:16:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742465_1641 (size=12151) 2024-12-02T22:16:24,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/6d8d23e800fe488ea47bd1093aab2a87 2024-12-02T22:16:24,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177844771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/b039ea6e1d19448ca6bb235b27a03c30 is 50, key is test_row_0/C:col10/1733177784724/Put/seqid=0 2024-12-02T22:16:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177844773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177844773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742466_1642 (size=12151) 2024-12-02T22:16:24,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/b039ea6e1d19448ca6bb235b27a03c30 2024-12-02T22:16:24,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/b097e72c934146da91d4a6839bdc92a7 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7 2024-12-02T22:16:24,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7, entries=200, sequenceid=168, filesize=38.8 K 2024-12-02T22:16:24,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/6d8d23e800fe488ea47bd1093aab2a87 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87 2024-12-02T22:16:24,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87, entries=150, sequenceid=168, filesize=11.9 K 2024-12-02T22:16:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/b039ea6e1d19448ca6bb235b27a03c30 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30 2024-12-02T22:16:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30, entries=150, sequenceid=168, filesize=11.9 K 2024-12-02T22:16:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a1bc7c7f9af7b8cbed3248a7876b3899 in 86ms, sequenceid=168, compaction requested=true 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T22:16:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T22:16:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:24,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:24,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d6c1877a100745f9844f2f6ee9e1ad0b_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:24,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742467_1643 (size=14794) 2024-12-02T22:16:24,897 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:24,903 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d6c1877a100745f9844f2f6ee9e1ad0b_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6c1877a100745f9844f2f6ee9e1ad0b_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:24,903 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ddbce353e87f402ea255db09987d651a, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:24,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ddbce353e87f402ea255db09987d651a is 175, key is test_row_0/A:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742468_1644 (size=39749) 2024-12-02T22:16:24,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ddbce353e87f402ea255db09987d651a 2024-12-02T22:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177844903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177844904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177844906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:24,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/58f3265e90ba45e69d98c9bbdbb30c28 is 50, key is test_row_0/B:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:24,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742469_1645 (size=12151) 2024-12-02T22:16:24,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/58f3265e90ba45e69d98c9bbdbb30c28 2024-12-02T22:16:24,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e90ba394a6e94daca2b2cf6b135e8727 is 50, key is test_row_0/C:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:24,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742470_1646 (size=12151) 2024-12-02T22:16:24,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e90ba394a6e94daca2b2cf6b135e8727 2024-12-02T22:16:24,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/ddbce353e87f402ea255db09987d651a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a 2024-12-02T22:16:24,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a, entries=200, sequenceid=189, filesize=38.8 K 2024-12-02T22:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/58f3265e90ba45e69d98c9bbdbb30c28 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28 2024-12-02T22:16:24,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28, entries=150, sequenceid=189, filesize=11.9 K 2024-12-02T22:16:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e90ba394a6e94daca2b2cf6b135e8727 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727 2024-12-02T22:16:24,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727, entries=150, sequenceid=189, filesize=11.9 K 2024-12-02T22:16:24,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a1bc7c7f9af7b8cbed3248a7876b3899 in 55ms, sequenceid=189, compaction requested=true 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-12-02T22:16:24,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:24,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-12-02T22:16:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T22:16:24,953 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:24,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-02T22:16:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:24,957 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:24,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:24,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:24,996 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/a768d70ce1a14b4da0a6da4ca0970d1f as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a768d70ce1a14b4da0a6da4ca0970d1f 2024-12-02T22:16:25,000 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into a768d70ce1a14b4da0a6da4ca0970d1f(size=12.2 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:25,001 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:25,001 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=12, startTime=1733177784544; duration=0sec 2024-12-02T22:16:25,001 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-12-02T22:16:25,001 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:25,001 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-12-02T22:16:25,002 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:25,002 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:25,002 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. because compaction request was cancelled 2024-12-02T22:16:25,002 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:25,002 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T22:16:25,003 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72762 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T22:16:25,004 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:25,004 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,004 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9b39dcaf37dd41198d80b5956695cbf0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=71.1 K 2024-12-02T22:16:25,004 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b39dcaf37dd41198d80b5956695cbf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733177778984 2024-12-02T22:16:25,005 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c771c6aac4544c1b4f3e42fe57c78f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733177780182 2024-12-02T22:16:25,005 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 21010efd874c43a9b2633623282d3481, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733177780728 2024-12-02T22:16:25,006 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e7d994b14f8403ca77bf70194d15a01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733177782926 2024-12-02T22:16:25,006 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b039ea6e1d19448ca6bb235b27a03c30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733177784091 2024-12-02T22:16:25,006 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e90ba394a6e94daca2b2cf6b135e8727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784764 2024-12-02T22:16:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:25,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T22:16:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:25,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:25,021 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/36a36d7965794549aa1dc15993463b3c as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c 2024-12-02T22:16:25,023 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#550 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:25,023 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/2b7849b7cef84ee88784b3491d926667 is 50, key is test_row_0/C:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:25,027 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 36a36d7965794549aa1dc15993463b3c(size=30.7 K), total size for store is 108.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:25,027 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=12, startTime=1733177784542; duration=0sec 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:25,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020207afad32ab414dae2684e42ecdd647_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177785016/Put/seqid=0 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. because compaction request was cancelled 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. because compaction request was cancelled 2024-12-02T22:16:25,027 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:25,028 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742471_1647 (size=12561) 2024-12-02T22:16:25,034 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:25,034 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:25,034 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,034 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a768d70ce1a14b4da0a6da4ca0970d1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=35.9 K 2024-12-02T22:16:25,035 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a768d70ce1a14b4da0a6da4ca0970d1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733177782926 2024-12-02T22:16:25,035 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d8d23e800fe488ea47bd1093aab2a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733177784091 2024-12-02T22:16:25,035 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f3265e90ba45e69d98c9bbdbb30c28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784764 2024-12-02T22:16:25,040 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/2b7849b7cef84ee88784b3491d926667 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/2b7849b7cef84ee88784b3491d926667 2024-12-02T22:16:25,043 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:25,044 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/541cf00aab10427fa7679b6bd42ee654 is 50, key is test_row_0/B:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:25,046 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 2b7849b7cef84ee88784b3491d926667(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:25,046 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=10, startTime=1733177784941; duration=0sec 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 3 compacting, 0 eligible, 16 blocking 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. because compaction request was cancelled 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:25,046 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:25,047 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:25,048 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:25,048 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,048 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=108.3 K 2024-12-02T22:16:25,048 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,048 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a] 2024-12-02T22:16:25,048 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a36d7965794549aa1dc15993463b3c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733177782926 2024-12-02T22:16:25,049 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting b097e72c934146da91d4a6839bdc92a7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733177784091 2024-12-02T22:16:25,049 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbce353e87f402ea255db09987d651a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784753 2024-12-02T22:16:25,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742472_1648 (size=17284) 2024-12-02T22:16:25,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:25,057 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:25,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020207afad32ab414dae2684e42ecdd647_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020207afad32ab414dae2684e42ecdd647_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:25,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fc787a5b59d34a88a8ddfd5b42cd8ba6, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:25,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 is 175, key is test_row_0/A:col10/1733177785016/Put/seqid=0 2024-12-02T22:16:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177845064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742473_1649 (size=12595) 2024-12-02T22:16:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177845065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177845066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,073 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/541cf00aab10427fa7679b6bd42ee654 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/541cf00aab10427fa7679b6bd42ee654 2024-12-02T22:16:25,077 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 541cf00aab10427fa7679b6bd42ee654(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:25,077 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:25,077 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177784811; duration=0sec 2024-12-02T22:16:25,077 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ef65b4747cb04303b720ccd5fa66491f_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:25,077 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:25,077 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:25,080 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ef65b4747cb04303b720ccd5fa66491f_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:25,080 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ef65b4747cb04303b720ccd5fa66491f_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:25,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742474_1650 (size=48389) 2024-12-02T22:16:25,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742475_1651 (size=4469) 2024-12-02T22:16:25,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177845173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177845173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177845183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177845383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177845383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177845393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 2024-12-02T22:16:25,490 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#553 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:25,490 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/71bd9b6164e04a419eeed83ae151ba39 is 175, key is test_row_0/A:col10/1733177784885/Put/seqid=0 2024-12-02T22:16:25,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/9015f2c0cff8478d80674582c26e91c9 is 50, key is test_row_0/B:col10/1733177785016/Put/seqid=0 2024-12-02T22:16:25,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742476_1652 (size=31549) 2024-12-02T22:16:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742477_1653 (size=12151) 2024-12-02T22:16:25,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/9015f2c0cff8478d80674582c26e91c9 2024-12-02T22:16:25,502 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/71bd9b6164e04a419eeed83ae151ba39 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39 2024-12-02T22:16:25,508 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 71bd9b6164e04a419eeed83ae151ba39(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:25,508 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:25,508 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177784811; duration=0sec 2024-12-02T22:16:25,508 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:25,508 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:25,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e54202297a45463ba9356e695ef19e33 is 50, key is test_row_0/C:col10/1733177785016/Put/seqid=0 2024-12-02T22:16:25,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742478_1654 (size=12151) 2024-12-02T22:16:25,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177845693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177845693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177845703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:25,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:25,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:25,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:25,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e54202297a45463ba9356e695ef19e33 2024-12-02T22:16:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 2024-12-02T22:16:25,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6, entries=250, sequenceid=203, filesize=47.3 K 2024-12-02T22:16:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/9015f2c0cff8478d80674582c26e91c9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9 2024-12-02T22:16:25,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9, entries=150, sequenceid=203, filesize=11.9 K 2024-12-02T22:16:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/e54202297a45463ba9356e695ef19e33 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33 2024-12-02T22:16:25,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33, entries=150, sequenceid=203, filesize=11.9 K 2024-12-02T22:16:25,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a1bc7c7f9af7b8cbed3248a7876b3899 in 914ms, sequenceid=203, compaction requested=false 2024-12-02T22:16:25,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:26,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:26,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d09761bdc1aa4bda82a0de81148b11d9_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177785064/Put/seqid=0 2024-12-02T22:16:26,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742479_1655 (size=12304) 2024-12-02T22:16:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:26,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177846219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177846222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177846224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177846333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177846333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177846333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:26,443 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d09761bdc1aa4bda82a0de81148b11d9_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d09761bdc1aa4bda82a0de81148b11d9_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/4b755b8b1ca14840b556b7a90d493165, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/4b755b8b1ca14840b556b7a90d493165 is 175, key is test_row_0/A:col10/1733177785064/Put/seqid=0 2024-12-02T22:16:26,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742480_1656 (size=31105) 2024-12-02T22:16:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177846543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177846543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177846543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/4b755b8b1ca14840b556b7a90d493165 2024-12-02T22:16:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/154a4aa494a6469b96a56f6a66a457e9 is 50, key is test_row_0/B:col10/1733177785064/Put/seqid=0 2024-12-02T22:16:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177846853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177846853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177846853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:26,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742481_1657 (size=12151) 2024-12-02T22:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:27,260 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/154a4aa494a6469b96a56f6a66a457e9 2024-12-02T22:16:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/a1347df9970645c2b2a2565617b45da1 is 50, key is test_row_0/C:col10/1733177785064/Put/seqid=0 2024-12-02T22:16:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742482_1658 (size=12151) 2024-12-02T22:16:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177847363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177847363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177847363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:27,673 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/a1347df9970645c2b2a2565617b45da1 2024-12-02T22:16:27,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/4b755b8b1ca14840b556b7a90d493165 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165 2024-12-02T22:16:27,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165, entries=150, sequenceid=226, filesize=30.4 K 2024-12-02T22:16:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/154a4aa494a6469b96a56f6a66a457e9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9 2024-12-02T22:16:27,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9, entries=150, sequenceid=226, filesize=11.9 K 2024-12-02T22:16:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/a1347df9970645c2b2a2565617b45da1 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1 2024-12-02T22:16:27,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1, entries=150, sequenceid=226, filesize=11.9 K 2024-12-02T22:16:27,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1662ms, sequenceid=226, compaction requested=true 2024-12-02T22:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-02T22:16:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-02T22:16:27,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-02T22:16:27,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-12-02T22:16:27,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.7400 sec 2024-12-02T22:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:28,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:28,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273d92984598347b287ce2addea3ec13d_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742483_1659 (size=14794) 2024-12-02T22:16:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177848408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177848408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177848410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177848410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177848410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177848523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177848523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177848523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177848523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177848523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177848733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177848733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177848733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177848734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177848734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:28,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:28,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273d92984598347b287ce2addea3ec13d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273d92984598347b287ce2addea3ec13d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:28,795 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:28,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 is 175, key is test_row_0/A:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:28,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742484_1660 (size=39749) 2024-12-02T22:16:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177849052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177849053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177849053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177849053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177849053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T22:16:29,124 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:29,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-02T22:16:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:29,133 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:29,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:29,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:29,200 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 2024-12-02T22:16:29,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/f9e24fc6055a496c8ae2a1e433fc8a9b is 50, key is test_row_0/B:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:29,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742485_1661 (size=12151) 2024-12-02T22:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:29,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:29,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:29,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:29,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:29,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:29,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177849563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177849563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177849574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177849574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177849574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:29,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/f9e24fc6055a496c8ae2a1e433fc8a9b 2024-12-02T22:16:29,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/396d1c391dca421fb13048bbe793e0cf is 50, key is test_row_0/C:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:29,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742486_1662 (size=12151) 2024-12-02T22:16:29,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:29,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:29,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:30,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/396d1c391dca421fb13048bbe793e0cf 2024-12-02T22:16:30,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 2024-12-02T22:16:30,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3, entries=200, sequenceid=240, filesize=38.8 K 2024-12-02T22:16:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/f9e24fc6055a496c8ae2a1e433fc8a9b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b 2024-12-02T22:16:30,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:30,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T22:16:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:30,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/396d1c391dca421fb13048bbe793e0cf as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf 2024-12-02T22:16:30,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T22:16:30,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1684ms, sequenceid=240, compaction requested=true 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:30,060 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:30,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:30,060 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:30,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:30,061 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:30,062 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,062 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/541cf00aab10427fa7679b6bd42ee654, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=47.9 K 2024-12-02T22:16:30,062 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150792 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:30,062 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:30,062 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,062 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=147.3 K 2024-12-02T22:16:30,062 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,062 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3] 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 541cf00aab10427fa7679b6bd42ee654, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784764 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bd9b6164e04a419eeed83ae151ba39, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784764 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9015f2c0cff8478d80674582c26e91c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733177784903 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc787a5b59d34a88a8ddfd5b42cd8ba6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733177784887 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 154a4aa494a6469b96a56f6a66a457e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733177785045 2024-12-02T22:16:30,063 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e24fc6055a496c8ae2a1e433fc8a9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786220 2024-12-02T22:16:30,064 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b755b8b1ca14840b556b7a90d493165, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733177785045 2024-12-02T22:16:30,064 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0f25b4268f4ff3a1d72aa9c0cba1d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786217 2024-12-02T22:16:30,090 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#562 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:30,090 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/6903c2e450f742aea0e70f7aab3d6e7e is 50, key is test_row_0/B:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:30,101 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,110 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120269f183ec303a47809f19879e0e98aff1_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,112 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120269f183ec303a47809f19879e0e98aff1_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,113 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120269f183ec303a47809f19879e0e98aff1_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742487_1663 (size=12731) 2024-12-02T22:16:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742488_1664 (size=4469) 2024-12-02T22:16:30,139 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#563 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:30,140 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/1fb9324d2cda411ca1eb2a83898171cd is 175, key is test_row_0/A:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:30,150 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/6903c2e450f742aea0e70f7aab3d6e7e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6903c2e450f742aea0e70f7aab3d6e7e 2024-12-02T22:16:30,154 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 6903c2e450f742aea0e70f7aab3d6e7e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:30,154 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:30,154 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=12, startTime=1733177790060; duration=0sec 2024-12-02T22:16:30,155 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:30,155 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:30,155 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:30,157 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:30,157 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:30,157 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,157 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/2b7849b7cef84ee88784b3491d926667, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=47.9 K 2024-12-02T22:16:30,158 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7849b7cef84ee88784b3491d926667, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733177784764 2024-12-02T22:16:30,159 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting e54202297a45463ba9356e695ef19e33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733177784903 2024-12-02T22:16:30,160 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a1347df9970645c2b2a2565617b45da1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733177785045 2024-12-02T22:16:30,161 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 396d1c391dca421fb13048bbe793e0cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786220 2024-12-02T22:16:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742489_1665 (size=31685) 2024-12-02T22:16:30,187 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#564 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:30,188 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9c71299fc06449fdbb3767150cc410a8 is 50, key is test_row_0/C:col10/1733177786223/Put/seqid=0 2024-12-02T22:16:30,192 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/1fb9324d2cda411ca1eb2a83898171cd as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd 2024-12-02T22:16:30,200 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 1fb9324d2cda411ca1eb2a83898171cd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:30,200 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:30,200 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=12, startTime=1733177790060; duration=0sec 2024-12-02T22:16:30,200 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:30,200 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:30,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T22:16:30,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,207 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:30,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742490_1666 (size=12697) 2024-12-02T22:16:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021e7ea093d9704575a179d27004e7a21f_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177788400/Put/seqid=0 2024-12-02T22:16:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742491_1667 (size=12354) 2024-12-02T22:16:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:30,281 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021e7ea093d9704575a179d27004e7a21f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e7ea093d9704575a179d27004e7a21f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/5e50e74bf67e479c916278a3b0902d6d, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/5e50e74bf67e479c916278a3b0902d6d is 175, key is test_row_0/A:col10/1733177788400/Put/seqid=0 2024-12-02T22:16:30,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742492_1668 (size=31155) 2024-12-02T22:16:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177850599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177850599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177850601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177850600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177850601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,643 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9c71299fc06449fdbb3767150cc410a8 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9c71299fc06449fdbb3767150cc410a8 2024-12-02T22:16:30,649 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 9c71299fc06449fdbb3767150cc410a8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:30,649 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:30,649 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=12, startTime=1733177790060; duration=0sec 2024-12-02T22:16:30,649 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:30,649 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:30,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/5e50e74bf67e479c916278a3b0902d6d 2024-12-02T22:16:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/a3df4ddfacff4cbfb9ea12c6e22e8641 is 50, key is test_row_0/B:col10/1733177788400/Put/seqid=0 2024-12-02T22:16:30,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742493_1669 (size=12201) 2024-12-02T22:16:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/a3df4ddfacff4cbfb9ea12c6e22e8641 2024-12-02T22:16:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177850713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177850713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177850713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177850714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177850714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/5bf5073ce6df4e70bf2c4bc945ab3c28 is 50, key is test_row_0/C:col10/1733177788400/Put/seqid=0 2024-12-02T22:16:30,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742494_1670 (size=12201) 2024-12-02T22:16:30,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/5bf5073ce6df4e70bf2c4bc945ab3c28 2024-12-02T22:16:30,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/5e50e74bf67e479c916278a3b0902d6d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d 2024-12-02T22:16:30,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d, entries=150, sequenceid=260, filesize=30.4 K 2024-12-02T22:16:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/a3df4ddfacff4cbfb9ea12c6e22e8641 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641 2024-12-02T22:16:30,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641, entries=150, sequenceid=260, filesize=11.9 K 2024-12-02T22:16:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/5bf5073ce6df4e70bf2c4bc945ab3c28 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28 2024-12-02T22:16:30,768 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28, entries=150, sequenceid=260, filesize=11.9 K 2024-12-02T22:16:30,769 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a1bc7c7f9af7b8cbed3248a7876b3899 in 562ms, sequenceid=260, compaction requested=false 2024-12-02T22:16:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:30,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:30,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-02T22:16:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-02T22:16:30,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-02T22:16:30,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6370 sec 2024-12-02T22:16:30,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6480 sec 2024-12-02T22:16:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:30,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:30,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:30,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026e6a91579ea9435b9907bab49c83f017_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:30,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742495_1671 (size=12454) 2024-12-02T22:16:30,942 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:30,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026e6a91579ea9435b9907bab49c83f017_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e6a91579ea9435b9907bab49c83f017_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:30,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fb3a7dfe90ed4da892966e09025f2e2e, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:30,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fb3a7dfe90ed4da892966e09025f2e2e is 175, key is test_row_0/A:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:30,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742496_1672 (size=31255) 2024-12-02T22:16:30,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177850952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177850952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177850957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177850961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177850962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177851063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177851073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177851073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177851073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177851074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T22:16:31,273 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:31,274 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-02T22:16:31,276 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:31,277 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:31,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177851273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177851284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177851284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177851284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177851284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fb3a7dfe90ed4da892966e09025f2e2e 2024-12-02T22:16:31,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5ec7dc0112854df9bee94b8f66ce1f1d is 50, key is test_row_0/B:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742497_1673 (size=12301) 2024-12-02T22:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:31,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:31,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:31,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:31,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177851583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:31,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177851593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177851594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177851594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177851594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:31,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5ec7dc0112854df9bee94b8f66ce1f1d 2024-12-02T22:16:31,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83f80d233ab14fed92af356cdf1382ec is 50, key is test_row_0/C:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:31,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742498_1674 (size=12301) 2024-12-02T22:16:31,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:31,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:32,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177852093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177852104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177852105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177852105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177852113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:32,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:32,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83f80d233ab14fed92af356cdf1382ec 2024-12-02T22:16:32,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/fb3a7dfe90ed4da892966e09025f2e2e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e 2024-12-02T22:16:32,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e, entries=150, sequenceid=277, filesize=30.5 K 2024-12-02T22:16:32,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/5ec7dc0112854df9bee94b8f66ce1f1d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d 2024-12-02T22:16:32,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d, entries=150, sequenceid=277, filesize=12.0 K 2024-12-02T22:16:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83f80d233ab14fed92af356cdf1382ec as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec 2024-12-02T22:16:32,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec, entries=150, sequenceid=277, filesize=12.0 K 2024-12-02T22:16:32,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1323ms, sequenceid=277, compaction requested=true 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:32,252 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:32,252 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:32,253 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:32,253 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,253 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=91.9 K 2024-12-02T22:16:32,253 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,253 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6903c2e450f742aea0e70f7aab3d6e7e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=36.4 K 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e] 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb9324d2cda411ca1eb2a83898171cd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786220 2024-12-02T22:16:32,253 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6903c2e450f742aea0e70f7aab3d6e7e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786220 2024-12-02T22:16:32,254 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e50e74bf67e479c916278a3b0902d6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733177788400 2024-12-02T22:16:32,254 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting a3df4ddfacff4cbfb9ea12c6e22e8641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733177788400 2024-12-02T22:16:32,254 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec7dc0112854df9bee94b8f66ce1f1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:32,254 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3a7dfe90ed4da892966e09025f2e2e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:32,263 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#571 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:32,263 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/68e138603dd84c4a99b69effe1ce70f3 is 50, key is test_row_0/B:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:32,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742499_1675 (size=12983) 2024-12-02T22:16:32,294 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:32,296 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202e9a40c705a4f46e59096ee9f43f9bcb1_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:32,298 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202e9a40c705a4f46e59096ee9f43f9bcb1_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:32,298 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e9a40c705a4f46e59096ee9f43f9bcb1_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:32,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742500_1676 (size=4469) 2024-12-02T22:16:32,348 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#572 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:32,349 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/1dcf6f4758e04050a483f3987d9b74e6 is 175, key is test_row_0/A:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:32,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:32,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:32,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120288961e243bef40d88e02fad7ec23b45b_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177790957/Put/seqid=0 2024-12-02T22:16:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742501_1677 (size=31937) 2024-12-02T22:16:32,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742502_1678 (size=12454) 2024-12-02T22:16:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120288961e243bef40d88e02fad7ec23b45b_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288961e243bef40d88e02fad7ec23b45b_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/084a648570694594ad32283064ea1625, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/084a648570694594ad32283064ea1625 is 175, key is test_row_0/A:col10/1733177790957/Put/seqid=0 2024-12-02T22:16:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:32,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742503_1679 (size=31255) 2024-12-02T22:16:32,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/084a648570694594ad32283064ea1625 2024-12-02T22:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bae3b6a2b72e4202823a77b12e831ba2 is 50, key is test_row_0/B:col10/1733177790957/Put/seqid=0 2024-12-02T22:16:32,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742504_1680 (size=12301) 2024-12-02T22:16:32,471 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bae3b6a2b72e4202823a77b12e831ba2 2024-12-02T22:16:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9435dd06c0a44f34bede246cafcd74ad is 50, key is test_row_0/C:col10/1733177790957/Put/seqid=0 2024-12-02T22:16:32,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742505_1681 (size=12301) 2024-12-02T22:16:32,699 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/68e138603dd84c4a99b69effe1ce70f3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/68e138603dd84c4a99b69effe1ce70f3 2024-12-02T22:16:32,704 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 68e138603dd84c4a99b69effe1ce70f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:32,704 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:32,704 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177792252; duration=0sec 2024-12-02T22:16:32,704 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:32,704 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:32,704 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:32,706 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:32,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:32,707 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,707 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9c71299fc06449fdbb3767150cc410a8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=36.3 K 2024-12-02T22:16:32,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c71299fc06449fdbb3767150cc410a8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733177786220 2024-12-02T22:16:32,707 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf5073ce6df4e70bf2c4bc945ab3c28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733177788400 2024-12-02T22:16:32,708 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f80d233ab14fed92af356cdf1382ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:32,723 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#576 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:32,723 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/cbbc90bdd3ef4be3a6ee07d36eeefabc is 50, key is test_row_0/C:col10/1733177790928/Put/seqid=0 2024-12-02T22:16:32,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742506_1682 (size=12949) 2024-12-02T22:16:32,763 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/cbbc90bdd3ef4be3a6ee07d36eeefabc as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/cbbc90bdd3ef4be3a6ee07d36eeefabc 2024-12-02T22:16:32,767 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into cbbc90bdd3ef4be3a6ee07d36eeefabc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:32,767 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:32,767 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=13, startTime=1733177792252; duration=0sec 2024-12-02T22:16:32,767 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:32,767 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:32,780 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/1dcf6f4758e04050a483f3987d9b74e6 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6 2024-12-02T22:16:32,785 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 1dcf6f4758e04050a483f3987d9b74e6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:32,785 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:32,785 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177792252; duration=0sec 2024-12-02T22:16:32,785 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:32,785 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:32,902 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9435dd06c0a44f34bede246cafcd74ad 2024-12-02T22:16:32,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/084a648570694594ad32283064ea1625 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625 2024-12-02T22:16:32,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625, entries=150, sequenceid=294, filesize=30.5 K 2024-12-02T22:16:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/bae3b6a2b72e4202823a77b12e831ba2 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2 2024-12-02T22:16:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,916 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T22:16:32,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/9435dd06c0a44f34bede246cafcd74ad as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad 2024-12-02T22:16:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T22:16:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for a1bc7c7f9af7b8cbed3248a7876b3899 in 572ms, sequenceid=294, compaction requested=false 2024-12-02T22:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-02T22:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-02T22:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-02T22:16:32,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-12-02T22:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.6540 sec 2024-12-02T22:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:33,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:33,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fc9b81cc085426f8b43da6c9bba32a1_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742507_1683 (size=22618) 2024-12-02T22:16:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177853207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177853207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177793185, max timestamp=1733177793210Num uniques 859; 2024-12-02T22:16:33,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177793185, max timestamp=1733177793210Num uniques 859; 2024-12-02T22:16:33,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733177793185, max timestamp=1733177793210Num uniques 859; 2024-12-02T22:16:33,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:33,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:33,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T22:16:33,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177853249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177853257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177853257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177853323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177853323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177853364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177853364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177853373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T22:16:33,424 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:33,424 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-02T22:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:33,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:33,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:33,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177853533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177853533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177853573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177853573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:33,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:33,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fc9b81cc085426f8b43da6c9bba32a1_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fc9b81cc085426f8b43da6c9bba32a1_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:33,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f5ce1f4841e84b0daab2fce7694787b9, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:33,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f5ce1f4841e84b0daab2fce7694787b9 is 175, key is test_row_0/A:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:33,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177853583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742508_1684 (size=66019) 2024-12-02T22:16:33,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:33,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:33,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:33,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177853843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177853843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:33,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177853883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177853883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177853893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:33,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f5ce1f4841e84b0daab2fce7694787b9 2024-12-02T22:16:33,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/08ac06b7921d41939004f684b70833ed is 50, key is test_row_0/B:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742509_1685 (size=9857) 2024-12-02T22:16:34,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:34,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177854353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177854353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177854393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177854393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:34,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177854403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:34,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/08ac06b7921d41939004f684b70833ed 2024-12-02T22:16:34,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83a823010b834398980ec24f551cb1f3 is 50, key is test_row_0/C:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:34,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742510_1686 (size=9857) 2024-12-02T22:16:34,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83a823010b834398980ec24f551cb1f3 2024-12-02T22:16:34,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/f5ce1f4841e84b0daab2fce7694787b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9 2024-12-02T22:16:34,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9, entries=350, sequenceid=307, filesize=64.5 K 2024-12-02T22:16:34,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/08ac06b7921d41939004f684b70833ed as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed 2024-12-02T22:16:34,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed, entries=100, sequenceid=307, filesize=9.6 K 2024-12-02T22:16:34,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/83a823010b834398980ec24f551cb1f3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3 2024-12-02T22:16:34,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3, entries=100, sequenceid=307, filesize=9.6 K 2024-12-02T22:16:34,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1272ms, sequenceid=307, compaction requested=true 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:34,435 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:34,435 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:34,436 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:34,436 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:34,436 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:34,436 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:34,436 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,436 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,436 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=126.2 K 2024-12-02T22:16:34,436 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/68e138603dd84c4a99b69effe1ce70f3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=34.3 K 2024-12-02T22:16:34,436 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,436 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9] 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e138603dd84c4a99b69effe1ce70f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dcf6f4758e04050a483f3987d9b74e6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084a648570694594ad32283064ea1625, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177790948 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting bae3b6a2b72e4202823a77b12e831ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177790948 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ce1f4841e84b0daab2fce7694787b9, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177793147 2024-12-02T22:16:34,437 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ac06b7921d41939004f684b70833ed, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177793161 2024-12-02T22:16:34,445 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:34,446 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#580 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:34,447 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/75c455b016f34fbda85127a31e36d629 is 50, key is test_row_0/B:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:34,448 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202947842bdbbd64ecdac62767c87ff5b77_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:34,451 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202947842bdbbd64ecdac62767c87ff5b77_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:34,451 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202947842bdbbd64ecdac62767c87ff5b77_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742511_1687 (size=13085) 2024-12-02T22:16:34,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742512_1688 (size=4469) 2024-12-02T22:16:34,476 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#581 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:34,476 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a13970d5b32743b8983bc2b2da3f703a is 175, key is test_row_0/A:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742513_1689 (size=32146) 2024-12-02T22:16:34,485 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/a13970d5b32743b8983bc2b2da3f703a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a 2024-12-02T22:16:34,492 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into a13970d5b32743b8983bc2b2da3f703a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:34,492 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:34,492 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177794435; duration=0sec 2024-12-02T22:16:34,493 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:34,493 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:34,493 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:34,494 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:34,495 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:34,495 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,495 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/cbbc90bdd3ef4be3a6ee07d36eeefabc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=34.3 K 2024-12-02T22:16:34,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:34,495 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbc90bdd3ef4be3a6ee07d36eeefabc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733177790598 2024-12-02T22:16:34,495 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:34,496 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9435dd06c0a44f34bede246cafcd74ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733177790948 2024-12-02T22:16:34,497 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a823010b834398980ec24f551cb1f3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177793161 2024-12-02T22:16:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eee5b1b4ab1b4d0397c2d44f507d379d_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177793254/Put/seqid=0 2024-12-02T22:16:34,506 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#583 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:34,506 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/44c0fb15035a4b278e37cd40aef64232 is 50, key is test_row_0/C:col10/1733177793162/Put/seqid=0 2024-12-02T22:16:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742514_1690 (size=13051) 2024-12-02T22:16:34,518 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/44c0fb15035a4b278e37cd40aef64232 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/44c0fb15035a4b278e37cd40aef64232 2024-12-02T22:16:34,524 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 44c0fb15035a4b278e37cd40aef64232(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:34,524 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:34,524 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=13, startTime=1733177794435; duration=0sec 2024-12-02T22:16:34,524 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:34,524 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:34,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742515_1691 (size=12454) 2024-12-02T22:16:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:34,861 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/75c455b016f34fbda85127a31e36d629 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/75c455b016f34fbda85127a31e36d629 2024-12-02T22:16:34,865 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 75c455b016f34fbda85127a31e36d629(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:34,865 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:34,865 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177794435; duration=0sec 2024-12-02T22:16:34,865 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:34,865 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:34,930 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eee5b1b4ab1b4d0397c2d44f507d379d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eee5b1b4ab1b4d0397c2d44f507d379d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c94b2a5f58784da3b112462cca9350ac, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:34,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c94b2a5f58784da3b112462cca9350ac is 175, key is test_row_0/A:col10/1733177793254/Put/seqid=0 2024-12-02T22:16:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742516_1692 (size=31255) 2024-12-02T22:16:35,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c94b2a5f58784da3b112462cca9350ac 2024-12-02T22:16:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/042c6a0f157e493e85a9e887d781b2b9 is 50, key is test_row_0/B:col10/1733177793254/Put/seqid=0 2024-12-02T22:16:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742517_1693 (size=12301) 2024-12-02T22:16:35,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/042c6a0f157e493e85a9e887d781b2b9 2024-12-02T22:16:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/657f79e4f2374323b947647815515469 is 50, key is test_row_0/C:col10/1733177793254/Put/seqid=0 2024-12-02T22:16:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742518_1694 (size=12301) 2024-12-02T22:16:35,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/657f79e4f2374323b947647815515469 2024-12-02T22:16:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:35,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c94b2a5f58784da3b112462cca9350ac as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac 2024-12-02T22:16:35,376 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac, entries=150, sequenceid=345, filesize=30.5 K 2024-12-02T22:16:35,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/042c6a0f157e493e85a9e887d781b2b9 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9 2024-12-02T22:16:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,381 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9, entries=150, sequenceid=345, filesize=12.0 K 2024-12-02T22:16:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/657f79e4f2374323b947647815515469 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469 2024-12-02T22:16:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469, entries=150, sequenceid=345, filesize=12.0 K 2024-12-02T22:16:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=33.54 KB/34350 for a1bc7c7f9af7b8cbed3248a7876b3899 in 891ms, sequenceid=345, compaction requested=false 2024-12-02T22:16:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-02T22:16:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-02T22:16:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-02T22:16:35,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9610 sec 2024-12-02T22:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(8855): Flush requested on a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:35,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.9670 sec 2024-12-02T22:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d9cb25dfd3d54f319aaf253a35fd85fb_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742519_1695 (size=20074) 2024-12-02T22:16:35,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:35,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d9cb25dfd3d54f319aaf253a35fd85fb_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9cb25dfd3d54f319aaf253a35fd85fb_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:35,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c7e550aae3f940d99a1a4da89b3b9860, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:35,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c7e550aae3f940d99a1a4da89b3b9860 is 175, key is test_row_0/A:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:35,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177795415, max timestamp=1733177795430Num uniques 859; 2024-12-02T22:16:35,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:35,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177795415, max timestamp=1733177795430Num uniques 859; 2024-12-02T22:16:35,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:35,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733177795415, max timestamp=1733177795430Num uniques 859; 2024-12-02T22:16:35,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7ae73638d47e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T22:16:35,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742520_1696 (size=57333) 2024-12-02T22:16:35,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c7e550aae3f940d99a1a4da89b3b9860 2024-12-02T22:16:35,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/db18e6d3de814702a14cba50cf2ecc2d is 50, key is test_row_0/B:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:35,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742521_1697 (size=12301) 2024-12-02T22:16:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177855459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177855462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177855464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177855465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T22:16:35,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:35,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T22:16:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-12-02T22:16:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T22:16:35,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T22:16:35,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T22:16:35,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T22:16:35,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177855573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177855573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177855573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177855573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177855573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T22:16:35,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T22:16:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:35,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177855783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177855783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177855783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177855784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177855784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:35,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/db18e6d3de814702a14cba50cf2ecc2d 2024-12-02T22:16:35,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/66a5ade9e4ed496cadf154cadc618a93 is 50, key is test_row_0/C:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:35,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742522_1698 (size=12301) 2024-12-02T22:16:35,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T22:16:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:35,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T22:16:36,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T22:16:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1733177856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52374 deadline: 1733177856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52366 deadline: 1733177856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52406 deadline: 1733177856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52388 deadline: 1733177856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 , the old value is region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T22:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., hostname=7ae73638d47e,41753,1733177627056, seqNum=5 because the exception is null or not the one we care about 2024-12-02T22:16:36,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T22:16:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. as already flushing 2024-12-02T22:16:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T22:16:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T22:16:36,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/66a5ade9e4ed496cadf154cadc618a93 2024-12-02T22:16:36,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/c7e550aae3f940d99a1a4da89b3b9860 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860 2024-12-02T22:16:36,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860, entries=300, sequenceid=357, filesize=56.0 K 2024-12-02T22:16:36,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/db18e6d3de814702a14cba50cf2ecc2d as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d 2024-12-02T22:16:36,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d, entries=150, sequenceid=357, filesize=12.0 K 2024-12-02T22:16:36,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/66a5ade9e4ed496cadf154cadc618a93 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93 2024-12-02T22:16:36,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93, entries=150, sequenceid=357, filesize=12.0 K 2024-12-02T22:16:36,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a1bc7c7f9af7b8cbed3248a7876b3899 in 889ms, sequenceid=357, compaction requested=true 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:36,282 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a1bc7c7f9af7b8cbed3248a7876b3899:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T22:16:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:36,282 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T22:16:36,283 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120734 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:36,283 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T22:16:36,283 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/A is initiating minor compaction (all files) 2024-12-02T22:16:36,283 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/B is initiating minor compaction (all files) 2024-12-02T22:16:36,283 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/A in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,283 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/B in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,283 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=117.9 K 2024-12-02T22:16:36,283 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/75c455b016f34fbda85127a31e36d629, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=36.8 K 2024-12-02T22:16:36,283 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,283 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. files: [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860] 2024-12-02T22:16:36,284 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c455b016f34fbda85127a31e36d629, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177790949 2024-12-02T22:16:36,284 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13970d5b32743b8983bc2b2da3f703a, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177790949 2024-12-02T22:16:36,284 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 042c6a0f157e493e85a9e887d781b2b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733177793247 2024-12-02T22:16:36,284 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting db18e6d3de814702a14cba50cf2ecc2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733177795382 2024-12-02T22:16:36,284 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94b2a5f58784da3b112462cca9350ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733177793247 2024-12-02T22:16:36,285 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e550aae3f940d99a1a4da89b3b9860, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733177795372 2024-12-02T22:16:36,293 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:36,293 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#B#compaction#589 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:36,294 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/1c9e9c411809408c96e03d47b78a8e06 is 50, key is test_row_0/B:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:36,304 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202bf8df1e9864f401b87d4adf6e66e94b5_a1bc7c7f9af7b8cbed3248a7876b3899 store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:36,307 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202bf8df1e9864f401b87d4adf6e66e94b5_a1bc7c7f9af7b8cbed3248a7876b3899, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:36,307 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bf8df1e9864f401b87d4adf6e66e94b5_a1bc7c7f9af7b8cbed3248a7876b3899 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:36,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742523_1699 (size=13187) 2024-12-02T22:16:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742524_1700 (size=4469) 2024-12-02T22:16:36,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T22:16:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T22:16:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d57f0d1dc00c4422a32f4898d97c26b3_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177795463/Put/seqid=0 2024-12-02T22:16:36,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742525_1701 (size=12454) 2024-12-02T22:16:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d57f0d1dc00c4422a32f4898d97c26b3_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d57f0d1dc00c4422a32f4898d97c26b3_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/09690e5c8c694b408467b4640bcdc359, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/09690e5c8c694b408467b4640bcdc359 is 175, key is test_row_0/A:col10/1733177795463/Put/seqid=0 2024-12-02T22:16:36,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742526_1702 (size=31255) 2024-12-02T22:16:36,378 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/09690e5c8c694b408467b4640bcdc359 2024-12-02T22:16:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/01b7d717dd144be39663a2dca31b7e72 is 50, key is test_row_0/B:col10/1733177795463/Put/seqid=0 2024-12-02T22:16:36,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742527_1703 (size=12301) 2024-12-02T22:16:36,389 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/01b7d717dd144be39663a2dca31b7e72 2024-12-02T22:16:36,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/fc789b53147a4fb4b5d2ac1930dbd763 is 50, key is test_row_0/C:col10/1733177795463/Put/seqid=0 2024-12-02T22:16:36,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742528_1704 (size=12301) 2024-12-02T22:16:36,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/fc789b53147a4fb4b5d2ac1930dbd763 2024-12-02T22:16:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/09690e5c8c694b408467b4640bcdc359 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/09690e5c8c694b408467b4640bcdc359 2024-12-02T22:16:36,406 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/09690e5c8c694b408467b4640bcdc359, entries=150, sequenceid=393, filesize=30.5 K 2024-12-02T22:16:36,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/01b7d717dd144be39663a2dca31b7e72 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/01b7d717dd144be39663a2dca31b7e72 2024-12-02T22:16:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/01b7d717dd144be39663a2dca31b7e72, entries=150, sequenceid=393, filesize=12.0 K 2024-12-02T22:16:36,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/fc789b53147a4fb4b5d2ac1930dbd763 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763 2024-12-02T22:16:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763, entries=150, sequenceid=393, filesize=12.0 K 2024-12-02T22:16:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for a1bc7c7f9af7b8cbed3248a7876b3899 in 79ms, sequenceid=393, compaction requested=true 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:36,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7ae73638d47e:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-12-02T22:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-02T22:16:36,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 843 msec 2024-12-02T22:16:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 848 msec 2024-12-02T22:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,480 INFO [Thread-2641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2641. 2024-12-02T22:16:36,480 DEBUG [Thread-2641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,480 DEBUG [Thread-2641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,480 INFO [Thread-2639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2639. 2024-12-02T22:16:36,480 DEBUG [Thread-2639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,480 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,480 DEBUG [Thread-2641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,480 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,498 INFO [Thread-2633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2633. 2024-12-02T22:16:36,498 DEBUG [Thread-2633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,498 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,498 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:36,510 INFO [Thread-2635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2635. 2024-12-02T22:16:36,510 DEBUG [Thread-2635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,510 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,510 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,511 INFO [Thread-2637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2637. 2024-12-02T22:16:36,511 DEBUG [Thread-2637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,511 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,511 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,605 INFO [Thread-2624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2624. 2024-12-02T22:16:36,605 INFO [Thread-2630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2630. 2024-12-02T22:16:36,605 INFO [Thread-2628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2628. 2024-12-02T22:16:36,605 DEBUG [Thread-2624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,605 DEBUG [Thread-2628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,605 DEBUG [Thread-2630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,605 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,607 INFO [Thread-2626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2626. 2024-12-02T22:16:36,607 DEBUG [Thread-2626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,607 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,607 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,614 INFO [Thread-2622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2622. 2024-12-02T22:16:36,614 DEBUG [Thread-2622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T22:16:36,614 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,614 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T22:16:36,713 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2884 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2859 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2888 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2872 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2863 2024-12-02T22:16:36,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T22:16:36,713 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:16:36,713 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:16:36,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:36,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:36,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T22:16:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:36,716 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177796716"}]},"ts":"1733177796716"} 2024-12-02T22:16:36,717 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T22:16:36,717 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T22:16:36,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T22:16:36,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, UNASSIGN}] 2024-12-02T22:16:36,720 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, UNASSIGN 2024-12-02T22:16:36,721 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=CLOSING, regionLocation=7ae73638d47e,41753,1733177627056 2024-12-02T22:16:36,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, UNASSIGN because future has completed 2024-12-02T22:16:36,722 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T22:16:36,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056}] 2024-12-02T22:16:36,722 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/1c9e9c411809408c96e03d47b78a8e06 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/1c9e9c411809408c96e03d47b78a8e06 2024-12-02T22:16:36,726 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/B of a1bc7c7f9af7b8cbed3248a7876b3899 into 1c9e9c411809408c96e03d47b78a8e06(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:36,726 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:36,726 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/B, priority=13, startTime=1733177796282; duration=0sec 2024-12-02T22:16:36,726 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T22:16:36,726 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:B 2024-12-02T22:16:36,726 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T22:16:36,727 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T22:16:36,727 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1541): a1bc7c7f9af7b8cbed3248a7876b3899/C is initiating minor compaction (all files) 2024-12-02T22:16:36,727 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a1bc7c7f9af7b8cbed3248a7876b3899/C in TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:36,727 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/44c0fb15035a4b278e37cd40aef64232, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763] into tmpdir=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp, totalSize=48.8 K 2024-12-02T22:16:36,728 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c0fb15035a4b278e37cd40aef64232, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733177790949 2024-12-02T22:16:36,728 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 657f79e4f2374323b947647815515469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733177793247 2024-12-02T22:16:36,728 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a5ade9e4ed496cadf154cadc618a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733177795382 2024-12-02T22:16:36,728 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] compactions.Compactor(225): Compacting fc789b53147a4fb4b5d2ac1930dbd763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733177795462 2024-12-02T22:16:36,731 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#A#compaction#590 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:36,731 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/015eb4ebe7154faf8e0dec06cc028e0b is 175, key is test_row_0/A:col10/1733177795391/Put/seqid=0 2024-12-02T22:16:36,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742529_1705 (size=32141) 2024-12-02T22:16:36,735 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a1bc7c7f9af7b8cbed3248a7876b3899#C#compaction#594 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T22:16:36,735 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/660f1959c03940c79bb249816b64f3b3 is 50, key is test_row_0/C:col10/1733177795463/Put/seqid=0 2024-12-02T22:16:36,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742530_1706 (size=13187) 2024-12-02T22:16:36,741 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/660f1959c03940c79bb249816b64f3b3 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/660f1959c03940c79bb249816b64f3b3 2024-12-02T22:16:36,744 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/C of a1bc7c7f9af7b8cbed3248a7876b3899 into 660f1959c03940c79bb249816b64f3b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:36,744 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:36,744 INFO [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/C, priority=12, startTime=1733177796282; duration=0sec 2024-12-02T22:16:36,744 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:36,744 DEBUG [RS:0;7ae73638d47e:41753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:C 2024-12-02T22:16:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:36,874 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:36,874 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T22:16:36,874 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing a1bc7c7f9af7b8cbed3248a7876b3899, disabling compactions & flushes 2024-12-02T22:16:36,874 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:37,138 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/015eb4ebe7154faf8e0dec06cc028e0b as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/015eb4ebe7154faf8e0dec06cc028e0b 2024-12-02T22:16:37,141 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a1bc7c7f9af7b8cbed3248a7876b3899/A of a1bc7c7f9af7b8cbed3248a7876b3899 into 015eb4ebe7154faf8e0dec06cc028e0b(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T22:16:37,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a1bc7c7f9af7b8cbed3248a7876b3899: 2024-12-02T22:16:37,141 INFO [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., storeName=a1bc7c7f9af7b8cbed3248a7876b3899/A, priority=13, startTime=1733177796282; duration=0sec 2024-12-02T22:16:37,141 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:37,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:37,141 DEBUG [RS:0;7ae73638d47e:41753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a1bc7c7f9af7b8cbed3248a7876b3899:A 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. after waiting 0 ms 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:37,141 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing a1bc7c7f9af7b8cbed3248a7876b3899 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=A 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=B 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a1bc7c7f9af7b8cbed3248a7876b3899, store=C 2024-12-02T22:16:37,141 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T22:16:37,146 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026c89d69ffa984b5ea42a8598a081461f_a1bc7c7f9af7b8cbed3248a7876b3899 is 50, key is test_row_0/A:col10/1733177796604/Put/seqid=0 2024-12-02T22:16:37,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742531_1707 (size=12454) 2024-12-02T22:16:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:37,549 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T22:16:37,551 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026c89d69ffa984b5ea42a8598a081461f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026c89d69ffa984b5ea42a8598a081461f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:37,552 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/20a78c15f659467f9e6d34c41012ea29, store: [table=TestAcidGuarantees family=A region=a1bc7c7f9af7b8cbed3248a7876b3899] 2024-12-02T22:16:37,553 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/20a78c15f659467f9e6d34c41012ea29 is 175, key is test_row_0/A:col10/1733177796604/Put/seqid=0 2024-12-02T22:16:37,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742532_1708 (size=31255) 2024-12-02T22:16:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:37,956 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/20a78c15f659467f9e6d34c41012ea29 2024-12-02T22:16:37,961 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 is 50, key is test_row_0/B:col10/1733177796604/Put/seqid=0 2024-12-02T22:16:37,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742533_1709 (size=12301) 2024-12-02T22:16:38,364 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 2024-12-02T22:16:38,369 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f9e3439176cf4987abb4d6bd2a146e8a is 50, key is test_row_0/C:col10/1733177796604/Put/seqid=0 2024-12-02T22:16:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742534_1710 (size=12301) 2024-12-02T22:16:38,772 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f9e3439176cf4987abb4d6bd2a146e8a 2024-12-02T22:16:38,775 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/A/20a78c15f659467f9e6d34c41012ea29 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/20a78c15f659467f9e6d34c41012ea29 2024-12-02T22:16:38,778 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/20a78c15f659467f9e6d34c41012ea29, entries=150, sequenceid=404, filesize=30.5 K 2024-12-02T22:16:38,778 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 2024-12-02T22:16:38,781 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29, entries=150, sequenceid=404, filesize=12.0 K 2024-12-02T22:16:38,781 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/.tmp/C/f9e3439176cf4987abb4d6bd2a146e8a as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f9e3439176cf4987abb4d6bd2a146e8a 2024-12-02T22:16:38,783 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f9e3439176cf4987abb4d6bd2a146e8a, entries=150, sequenceid=404, filesize=12.0 K 2024-12-02T22:16:38,784 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1643ms, sequenceid=404, compaction requested=true 2024-12-02T22:16:38,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860] to archive 2024-12-02T22:16:38,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:38,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/3475b951c16649b6b96c8f30999dacc8 2024-12-02T22:16:38,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a230bfc0fa7e455b906752fe8fa93b01 2024-12-02T22:16:38,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6f92d65a1dfb4aada371ebba98b5d3e2 2024-12-02T22:16:38,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/720e477cc88e40788e95c7a507b3cda8 2024-12-02T22:16:38,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/6519417f3ee640cea8e04bfee7bcd0bf 2024-12-02T22:16:38,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a26eb72f1c5c4a12ba0c35a6919a3036 2024-12-02T22:16:38,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/e4aa3ec13f3d4d7f9d622b7aa5e61a0d 2024-12-02T22:16:38,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f226078312a24aa2981ff18d29c71791 2024-12-02T22:16:38,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/93749ea02c9741f6a161301c50abbc4f 2024-12-02T22:16:38,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ab1e087faa7d41de80b2cbcad70f05bf 2024-12-02T22:16:38,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/36a36d7965794549aa1dc15993463b3c 2024-12-02T22:16:38,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/b097e72c934146da91d4a6839bdc92a7 2024-12-02T22:16:38,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/ddbce353e87f402ea255db09987d651a 2024-12-02T22:16:38,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/71bd9b6164e04a419eeed83ae151ba39 2024-12-02T22:16:38,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fc787a5b59d34a88a8ddfd5b42cd8ba6 2024-12-02T22:16:38,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/4b755b8b1ca14840b556b7a90d493165 2024-12-02T22:16:38,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/bc0f25b4268f4ff3a1d72aa9c0cba1d3 2024-12-02T22:16:38,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1fb9324d2cda411ca1eb2a83898171cd 2024-12-02T22:16:38,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/5e50e74bf67e479c916278a3b0902d6d 2024-12-02T22:16:38,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/1dcf6f4758e04050a483f3987d9b74e6 2024-12-02T22:16:38,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/fb3a7dfe90ed4da892966e09025f2e2e 2024-12-02T22:16:38,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/084a648570694594ad32283064ea1625 2024-12-02T22:16:38,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/f5ce1f4841e84b0daab2fce7694787b9 2024-12-02T22:16:38,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/a13970d5b32743b8983bc2b2da3f703a 2024-12-02T22:16:38,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c94b2a5f58784da3b112462cca9350ac 2024-12-02T22:16:38,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/c7e550aae3f940d99a1a4da89b3b9860 2024-12-02T22:16:38,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c434b2cce16b4bd499471d49bf2be7cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/3ce292a29a554e72aa3a01c80af6baee, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a768d70ce1a14b4da0a6da4ca0970d1f, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/541cf00aab10427fa7679b6bd42ee654, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6903c2e450f742aea0e70f7aab3d6e7e, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/68e138603dd84c4a99b69effe1ce70f3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/75c455b016f34fbda85127a31e36d629, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d] to archive 2024-12-02T22:16:38,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:38,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/8dbde260e3f64e41ad5faf9497b12776 2024-12-02T22:16:38,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/386f35a0bdf5482fa983fbd139e502d8 2024-12-02T22:16:38,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c434b2cce16b4bd499471d49bf2be7cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c434b2cce16b4bd499471d49bf2be7cf 2024-12-02T22:16:38,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5b613377206a4780aa6bfc6674f66d14 2024-12-02T22:16:38,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/c3242c19c2764569a1d9514cef400afb 2024-12-02T22:16:38,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/3ce292a29a554e72aa3a01c80af6baee to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/3ce292a29a554e72aa3a01c80af6baee 2024-12-02T22:16:38,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bb2920b964cc4a92abc3c78368978ebc 2024-12-02T22:16:38,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/e4c1c97a2bfe452fae1934482597c66c 2024-12-02T22:16:38,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/378203cb233347958e7a6da754bb6ad4 2024-12-02T22:16:38,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a768d70ce1a14b4da0a6da4ca0970d1f to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a768d70ce1a14b4da0a6da4ca0970d1f 2024-12-02T22:16:38,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/fc63d6bbe30743e99d1151a9f914c82e 2024-12-02T22:16:38,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6d8d23e800fe488ea47bd1093aab2a87 2024-12-02T22:16:38,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/541cf00aab10427fa7679b6bd42ee654 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/541cf00aab10427fa7679b6bd42ee654 2024-12-02T22:16:38,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/58f3265e90ba45e69d98c9bbdbb30c28 2024-12-02T22:16:38,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/9015f2c0cff8478d80674582c26e91c9 2024-12-02T22:16:38,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/154a4aa494a6469b96a56f6a66a457e9 2024-12-02T22:16:38,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6903c2e450f742aea0e70f7aab3d6e7e to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/6903c2e450f742aea0e70f7aab3d6e7e 2024-12-02T22:16:38,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/f9e24fc6055a496c8ae2a1e433fc8a9b 2024-12-02T22:16:38,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/a3df4ddfacff4cbfb9ea12c6e22e8641 2024-12-02T22:16:38,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/68e138603dd84c4a99b69effe1ce70f3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/68e138603dd84c4a99b69effe1ce70f3 2024-12-02T22:16:38,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/5ec7dc0112854df9bee94b8f66ce1f1d 2024-12-02T22:16:38,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/bae3b6a2b72e4202823a77b12e831ba2 2024-12-02T22:16:38,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/75c455b016f34fbda85127a31e36d629 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/75c455b016f34fbda85127a31e36d629 2024-12-02T22:16:38,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/08ac06b7921d41939004f684b70833ed 2024-12-02T22:16:38,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/042c6a0f157e493e85a9e887d781b2b9 2024-12-02T22:16:38,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/db18e6d3de814702a14cba50cf2ecc2d 2024-12-02T22:16:38,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c0d9f19fa0b4f188859868463917560, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9b39dcaf37dd41198d80b5956695cbf0, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/2b7849b7cef84ee88784b3491d926667, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9c71299fc06449fdbb3767150cc410a8, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/cbbc90bdd3ef4be3a6ee07d36eeefabc, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/44c0fb15035a4b278e37cd40aef64232, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763] to archive 2024-12-02T22:16:38,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T22:16:38,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f20ef912685b4d009f04775f313789b9 2024-12-02T22:16:38,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/04a44e752e104031b38dab5d6f033bb8 2024-12-02T22:16:38,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c0d9f19fa0b4f188859868463917560 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c0d9f19fa0b4f188859868463917560 2024-12-02T22:16:38,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/74772448204140e4a90af9fa03365492 2024-12-02T22:16:38,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c7006de0af34ec4904f4ed29f122d7c 2024-12-02T22:16:38,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9b39dcaf37dd41198d80b5956695cbf0 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9b39dcaf37dd41198d80b5956695cbf0 2024-12-02T22:16:38,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e5b9edd6ecf34c87a708cf6de82d4058 2024-12-02T22:16:38,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1c771c6aac4544c1b4f3e42fe57c78f4 2024-12-02T22:16:38,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/21010efd874c43a9b2633623282d3481 2024-12-02T22:16:38,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/1e7d994b14f8403ca77bf70194d15a01 2024-12-02T22:16:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/b039ea6e1d19448ca6bb235b27a03c30 2024-12-02T22:16:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/2b7849b7cef84ee88784b3491d926667 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/2b7849b7cef84ee88784b3491d926667 2024-12-02T22:16:38,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e90ba394a6e94daca2b2cf6b135e8727 2024-12-02T22:16:38,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/e54202297a45463ba9356e695ef19e33 2024-12-02T22:16:38,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/a1347df9970645c2b2a2565617b45da1 2024-12-02T22:16:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9c71299fc06449fdbb3767150cc410a8 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9c71299fc06449fdbb3767150cc410a8 2024-12-02T22:16:38,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/396d1c391dca421fb13048bbe793e0cf 2024-12-02T22:16:38,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/5bf5073ce6df4e70bf2c4bc945ab3c28 2024-12-02T22:16:38,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/cbbc90bdd3ef4be3a6ee07d36eeefabc to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/cbbc90bdd3ef4be3a6ee07d36eeefabc 2024-12-02T22:16:38,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83f80d233ab14fed92af356cdf1382ec 2024-12-02T22:16:38,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/9435dd06c0a44f34bede246cafcd74ad 2024-12-02T22:16:38,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/44c0fb15035a4b278e37cd40aef64232 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/44c0fb15035a4b278e37cd40aef64232 2024-12-02T22:16:38,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/83a823010b834398980ec24f551cb1f3 2024-12-02T22:16:38,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/657f79e4f2374323b947647815515469 2024-12-02T22:16:38,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/66a5ade9e4ed496cadf154cadc618a93 2024-12-02T22:16:38,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/fc789b53147a4fb4b5d2ac1930dbd763 2024-12-02T22:16:38,854 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-12-02T22:16:38,855 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. 2024-12-02T22:16:38,855 DEBUG [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for a1bc7c7f9af7b8cbed3248a7876b3899: Waiting for close lock at 1733177796874Running coprocessor pre-close hooks at 1733177796874Disabling compacts and flushes for region at 1733177796874Disabling writes for close at 1733177797141 (+267 ms)Obtaining lock to block concurrent updates at 1733177797141Preparing flush snapshotting stores in a1bc7c7f9af7b8cbed3248a7876b3899 at 1733177797141Finished memstore snapshotting TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733177797141Flushing stores of TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899. at 1733177797142 (+1 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/A: creating writer at 1733177797142Flushing mob file a1bc7c7f9af7b8cbed3248a7876b3899/A: appending metadata at 1733177797145 (+3 ms)Flushing mob file a1bc7c7f9af7b8cbed3248a7876b3899/A: closing flushed file at 1733177797145Flushing a1bc7c7f9af7b8cbed3248a7876b3899/A: appending metadata at 1733177797552 (+407 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/A: closing flushed file at 1733177797552Flushing a1bc7c7f9af7b8cbed3248a7876b3899/B: creating writer at 1733177797959 (+407 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/B: appending metadata at 1733177797960 (+1 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/B: closing flushed file at 1733177797960Flushing a1bc7c7f9af7b8cbed3248a7876b3899/C: creating writer at 1733177798367 (+407 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/C: appending metadata at 1733177798369 (+2 ms)Flushing a1bc7c7f9af7b8cbed3248a7876b3899/C: closing flushed file at 1733177798369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d522b15: reopening flushed file at 1733177798775 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ff8ea: reopening flushed file at 1733177798778 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71ccede4: reopening flushed file at 1733177798781 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a1bc7c7f9af7b8cbed3248a7876b3899 in 1643ms, sequenceid=404, compaction requested=true at 1733177798784 (+3 ms)Writing region close event to WAL at 1733177798852 (+68 ms)Running coprocessor post-close hooks at 1733177798855 (+3 ms)Closed at 1733177798855 2024-12-02T22:16:38,857 INFO [RS_CLOSE_REGION-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:38,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=a1bc7c7f9af7b8cbed3248a7876b3899, regionState=CLOSED 2024-12-02T22:16:38,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 because future has completed 2024-12-02T22:16:38,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-12-02T22:16:38,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure a1bc7c7f9af7b8cbed3248a7876b3899, server=7ae73638d47e,41753,1733177627056 in 2.1370 sec 2024-12-02T22:16:38,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-12-02T22:16:38,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a1bc7c7f9af7b8cbed3248a7876b3899, UNASSIGN in 2.1410 sec 2024-12-02T22:16:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:38,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-02T22:16:38,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1440 sec 2024-12-02T22:16:38,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733177798864"}]},"ts":"1733177798864"} 2024-12-02T22:16:38,865 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T22:16:38,865 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T22:16:38,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1520 sec 2024-12-02T22:16:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T22:16:40,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:40,884 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T22:16:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,885 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T22:16:40,886 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,888 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C, FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits] 2024-12-02T22:16:40,892 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/015eb4ebe7154faf8e0dec06cc028e0b to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/015eb4ebe7154faf8e0dec06cc028e0b 2024-12-02T22:16:40,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/09690e5c8c694b408467b4640bcdc359 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/09690e5c8c694b408467b4640bcdc359 2024-12-02T22:16:40,894 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/20a78c15f659467f9e6d34c41012ea29 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/A/20a78c15f659467f9e6d34c41012ea29 2024-12-02T22:16:40,895 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/01b7d717dd144be39663a2dca31b7e72 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/01b7d717dd144be39663a2dca31b7e72 2024-12-02T22:16:40,896 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/1c9e9c411809408c96e03d47b78a8e06 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/1c9e9c411809408c96e03d47b78a8e06 2024-12-02T22:16:40,897 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/B/7a3aa9e1d51a4ca186e52ee1ee6f2e29 2024-12-02T22:16:40,899 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/660f1959c03940c79bb249816b64f3b3 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/660f1959c03940c79bb249816b64f3b3 2024-12-02T22:16:40,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f9e3439176cf4987abb4d6bd2a146e8a to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/C/f9e3439176cf4987abb4d6bd2a146e8a 2024-12-02T22:16:40,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits/407.seqid to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899/recovered.edits/407.seqid 2024-12-02T22:16:40,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/default/TestAcidGuarantees/a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T22:16:40,903 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:16:40,903 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T22:16:40,905 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020207afad32ab414dae2684e42ecdd647_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020207afad32ab414dae2684e42ecdd647_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,906 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120204b788909035427dbbebe3ccd851fa95_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120204b788909035427dbbebe3ccd851fa95_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,907 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202192330cd998b47f1981106e2250b83d4_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202192330cd998b47f1981106e2250b83d4_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,908 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e7ea093d9704575a179d27004e7a21f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e7ea093d9704575a179d27004e7a21f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,908 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202383ac636fbde44a9a9363fee8d92cb3d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202383ac636fbde44a9a9363fee8d92cb3d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,909 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202694e5829087743e7882898a437b17486_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202694e5829087743e7882898a437b17486_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,910 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026c89d69ffa984b5ea42a8598a081461f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026c89d69ffa984b5ea42a8598a081461f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,911 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e6a91579ea9435b9907bab49c83f017_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e6a91579ea9435b9907bab49c83f017_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,912 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273651747ee8a4785a8c3beaa76ded420_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273651747ee8a4785a8c3beaa76ded420_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,913 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273d92984598347b287ce2addea3ec13d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273d92984598347b287ce2addea3ec13d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,913 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fc9b81cc085426f8b43da6c9bba32a1_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fc9b81cc085426f8b43da6c9bba32a1_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,914 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288961e243bef40d88e02fad7ec23b45b_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288961e243bef40d88e02fad7ec23b45b_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,915 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028b447ff15a7e46459c1b4d52a1a596ba_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028b447ff15a7e46459c1b4d52a1a596ba_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,916 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202af2e3294ad4e4e69bf50bddc62ca381d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202af2e3294ad4e4e69bf50bddc62ca381d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,916 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bfc2f964aa52482ab2c868fdca71386f_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bfc2f964aa52482ab2c868fdca71386f_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,917 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0de651d68ef424aa6ec482ab412d54e_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0de651d68ef424aa6ec482ab412d54e_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,918 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d09761bdc1aa4bda82a0de81148b11d9_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d09761bdc1aa4bda82a0de81148b11d9_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,919 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d57f0d1dc00c4422a32f4898d97c26b3_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d57f0d1dc00c4422a32f4898d97c26b3_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6c1877a100745f9844f2f6ee9e1ad0b_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6c1877a100745f9844f2f6ee9e1ad0b_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9cb25dfd3d54f319aaf253a35fd85fb_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9cb25dfd3d54f319aaf253a35fd85fb_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eee5b1b4ab1b4d0397c2d44f507d379d_a1bc7c7f9af7b8cbed3248a7876b3899 to hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eee5b1b4ab1b4d0397c2d44f507d379d_a1bc7c7f9af7b8cbed3248a7876b3899 2024-12-02T22:16:40,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T22:16:40,923 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,925 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T22:16:40,926 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T22:16:40,927 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,927 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T22:16:40,927 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733177800927"}]},"ts":"9223372036854775807"} 2024-12-02T22:16:40,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T22:16:40,929 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a1bc7c7f9af7b8cbed3248a7876b3899, NAME => 'TestAcidGuarantees,,1733177773361.a1bc7c7f9af7b8cbed3248a7876b3899.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T22:16:40,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T22:16:40,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733177800929"}]},"ts":"9223372036854775807"} 2024-12-02T22:16:40,930 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T22:16:40,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T22:16:40,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-12-02T22:16:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36511 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T22:16:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T22:16:40,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T22:16:41,003 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=569 (was 471) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7140 (was 7215) 2024-12-02T22:16:41,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T22:16:41,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:16:41,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:16:41,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:41,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:41,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T22:16:41,003 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T22:16:41,003 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=974344401, stopped=false 2024-12-02T22:16:41,004 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7ae73638d47e,36511,1733177626309 2024-12-02T22:16:41,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T22:16:41,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T22:16:41,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:16:41,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:16:41,006 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T22:16:41,006 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T22:16:41,007 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:16:41,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:41,007 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7ae73638d47e,41753,1733177627056' ***** 2024-12-02T22:16:41,007 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T22:16:41,007 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:16:41,008 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T22:16:41,008 INFO [RS:0;7ae73638d47e:41753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T22:16:41,008 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T22:16:41,008 INFO [RS:0;7ae73638d47e:41753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T22:16:41,008 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(959): stopping server 7ae73638d47e,41753,1733177627056 2024-12-02T22:16:41,008 INFO [RS:0;7ae73638d47e:41753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:16:41,009 INFO [RS:0;7ae73638d47e:41753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7ae73638d47e:41753. 2024-12-02T22:16:41,009 DEBUG [RS:0;7ae73638d47e:41753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T22:16:41,009 DEBUG [RS:0;7ae73638d47e:41753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:41,009 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T22:16:41,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T22:16:41,012 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T22:16:41,012 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T22:16:41,013 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T22:16:41,013 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T22:16:41,013 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T22:16:41,013 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T22:16:41,014 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T22:16:41,014 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:41,014 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T22:16:41,014 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T22:16:41,014 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T22:16:41,014 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T22:16:41,035 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/info/2c641340840a4ed0b7182652df4b4d4e is 95, key is TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78./info:/1733177656174/DeleteFamily/seqid=0 2024-12-02T22:16:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742535_1711 (size=6514) 2024-12-02T22:16:41,074 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7ae73638d47e:36511 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-12-02T22:16:41,077 INFO [regionserver/7ae73638d47e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:16:41,214 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:41,227 INFO [regionserver/7ae73638d47e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T22:16:41,227 INFO [regionserver/7ae73638d47e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T22:16:41,414 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:41,438 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/info/2c641340840a4ed0b7182652df4b4d4e 2024-12-02T22:16:41,457 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/ns/0ccd4f3a36c84eeb88ca56e40c38ad83 is 93, key is TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78./ns:/1733177656129/DeleteFamily/seqid=0 2024-12-02T22:16:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742536_1712 (size=5906) 2024-12-02T22:16:41,614 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:41,814 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:41,860 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/ns/0ccd4f3a36c84eeb88ca56e40c38ad83 2024-12-02T22:16:41,877 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/rep_barrier/862999fa371c409eb29311e787fed9b5 is 102, key is TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78./rep_barrier:/1733177656129/DeleteFamily/seqid=0 2024-12-02T22:16:41,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742537_1713 (size=6025) 2024-12-02T22:16:42,014 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:42,215 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:42,281 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/rep_barrier/862999fa371c409eb29311e787fed9b5 2024-12-02T22:16:42,299 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/table/a0fe857b913e4068a784fe06654c6ebe is 96, key is TestAcidGuarantees,,1733177629384.8de9171fd3e9b964cfb173aae742ae78./table:/1733177656129/DeleteFamily/seqid=0 2024-12-02T22:16:42,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742538_1714 (size=5936) 2024-12-02T22:16:42,415 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:42,615 DEBUG [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T22:16:42,702 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/table/a0fe857b913e4068a784fe06654c6ebe 2024-12-02T22:16:42,705 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/info/2c641340840a4ed0b7182652df4b4d4e as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/info/2c641340840a4ed0b7182652df4b4d4e 2024-12-02T22:16:42,708 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/info/2c641340840a4ed0b7182652df4b4d4e, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T22:16:42,709 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/ns/0ccd4f3a36c84eeb88ca56e40c38ad83 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/ns/0ccd4f3a36c84eeb88ca56e40c38ad83 2024-12-02T22:16:42,711 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/ns/0ccd4f3a36c84eeb88ca56e40c38ad83, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T22:16:42,712 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/rep_barrier/862999fa371c409eb29311e787fed9b5 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/rep_barrier/862999fa371c409eb29311e787fed9b5 2024-12-02T22:16:42,714 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/rep_barrier/862999fa371c409eb29311e787fed9b5, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T22:16:42,715 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/.tmp/table/a0fe857b913e4068a784fe06654c6ebe as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/table/a0fe857b913e4068a784fe06654c6ebe 2024-12-02T22:16:42,717 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/table/a0fe857b913e4068a784fe06654c6ebe, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T22:16:42,718 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-12-02T22:16:42,722 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T22:16:42,722 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T22:16:42,722 INFO [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T22:16:42,722 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733177801013Running coprocessor pre-close hooks at 1733177801013Disabling compacts and flushes for region at 1733177801013Disabling writes for close at 1733177801014 (+1 ms)Obtaining lock to block concurrent updates at 1733177801014Preparing flush snapshotting stores in 1588230740 at 1733177801014Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733177801014Flushing stores of hbase:meta,,1.1588230740 at 1733177801015 (+1 ms)Flushing 1588230740/info: creating writer at 1733177801015Flushing 1588230740/info: appending metadata at 1733177801035 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733177801035Flushing 1588230740/ns: creating writer at 1733177801442 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733177801456 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733177801456Flushing 1588230740/rep_barrier: creating writer at 1733177801863 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733177801877 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733177801877Flushing 1588230740/table: creating writer at 1733177802284 (+407 ms)Flushing 1588230740/table: appending metadata at 1733177802298 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733177802298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@228e083b: reopening flushed file at 1733177802705 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36dfe6a6: reopening flushed file at 1733177802708 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f77ea5: reopening flushed file at 1733177802711 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6ceb4b: reopening flushed file at 1733177802714 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1733177802718 (+4 ms)Writing region close event to WAL at 1733177802719 (+1 ms)Running coprocessor post-close hooks at 1733177802722 (+3 ms)Closed at 1733177802722 2024-12-02T22:16:42,722 DEBUG [RS_CLOSE_META-regionserver/7ae73638d47e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T22:16:42,815 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(976): stopping server 7ae73638d47e,41753,1733177627056; all regions closed. 2024-12-02T22:16:42,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741834_1010 (size=25344) 2024-12-02T22:16:42,822 DEBUG [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/oldWALs 2024-12-02T22:16:42,822 INFO [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ae73638d47e%2C41753%2C1733177627056.meta:.meta(num 1733177628956) 2024-12-02T22:16:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741833_1009 (size=18712286) 2024-12-02T22:16:42,825 DEBUG [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/oldWALs 2024-12-02T22:16:42,825 INFO [RS:0;7ae73638d47e:41753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7ae73638d47e%2C41753%2C1733177627056:(num 1733177628382) 2024-12-02T22:16:42,825 DEBUG [RS:0;7ae73638d47e:41753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T22:16:42,825 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T22:16:42,825 INFO [RS:0;7ae73638d47e:41753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:16:42,826 INFO [RS:0;7ae73638d47e:41753 {}] hbase.ChoreService(370): Chore service for: regionserver/7ae73638d47e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T22:16:42,826 INFO [RS:0;7ae73638d47e:41753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:16:42,826 INFO [regionserver/7ae73638d47e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:16:42,826 INFO [RS:0;7ae73638d47e:41753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41753 2024-12-02T22:16:42,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7ae73638d47e,41753,1733177627056 2024-12-02T22:16:42,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T22:16:42,831 INFO [RS:0;7ae73638d47e:41753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:16:42,832 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7ae73638d47e,41753,1733177627056] 2024-12-02T22:16:42,833 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7ae73638d47e,41753,1733177627056 already deleted, retry=false 2024-12-02T22:16:42,833 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7ae73638d47e,41753,1733177627056 expired; onlineServers=0 2024-12-02T22:16:42,833 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7ae73638d47e,36511,1733177626309' ***** 2024-12-02T22:16:42,834 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T22:16:42,834 INFO [M:0;7ae73638d47e:36511 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T22:16:42,834 INFO [M:0;7ae73638d47e:36511 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T22:16:42,834 DEBUG [M:0;7ae73638d47e:36511 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T22:16:42,834 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T22:16:42,834 DEBUG [M:0;7ae73638d47e:36511 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T22:16:42,834 DEBUG [master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733177628086 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.large.0-1733177628086,5,FailOnTimeoutGroup] 2024-12-02T22:16:42,834 DEBUG [master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733177628088 {}] cleaner.HFileCleaner(306): Exit Thread[master/7ae73638d47e:0:becomeActiveMaster-HFileCleaner.small.0-1733177628088,5,FailOnTimeoutGroup] 2024-12-02T22:16:42,834 INFO [M:0;7ae73638d47e:36511 {}] hbase.ChoreService(370): Chore service for: master/7ae73638d47e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T22:16:42,834 INFO [M:0;7ae73638d47e:36511 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T22:16:42,834 DEBUG [M:0;7ae73638d47e:36511 {}] master.HMaster(1795): Stopping service threads 2024-12-02T22:16:42,835 INFO [M:0;7ae73638d47e:36511 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T22:16:42,835 INFO [M:0;7ae73638d47e:36511 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T22:16:42,835 ERROR [M:0;7ae73638d47e:36511 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:34911 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34911,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T22:16:42,835 INFO [M:0;7ae73638d47e:36511 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T22:16:42,836 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T22:16:42,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T22:16:42,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T22:16:42,836 DEBUG [M:0;7ae73638d47e:36511 {}] zookeeper.ZKUtil(347): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T22:16:42,836 WARN [M:0;7ae73638d47e:36511 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T22:16:42,837 INFO [M:0;7ae73638d47e:36511 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/.lastflushedseqids 2024-12-02T22:16:42,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742539_1715 (size=72) 2024-12-02T22:16:42,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:16:42,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x1009d0dd6140001, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:16:42,934 INFO [RS:0;7ae73638d47e:41753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:16:42,934 INFO [RS:0;7ae73638d47e:41753 {}] regionserver.HRegionServer(1031): Exiting; stopping=7ae73638d47e,41753,1733177627056; zookeeper connection closed. 2024-12-02T22:16:42,935 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73a94f64 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73a94f64 2024-12-02T22:16:42,935 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T22:16:43,247 INFO [M:0;7ae73638d47e:36511 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T22:16:43,248 INFO [M:0;7ae73638d47e:36511 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T22:16:43,248 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T22:16:43,248 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:16:43,248 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:16:43,248 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T22:16:43,248 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:16:43,248 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=781.49 KB heapSize=957.39 KB 2024-12-02T22:16:43,263 DEBUG [M:0;7ae73638d47e:36511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/678c9ca8603141709df6937cfe362b59 is 82, key is hbase:meta,,1/info:regioninfo/1733177629049/Put/seqid=0 2024-12-02T22:16:43,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742540_1716 (size=5672) 2024-12-02T22:16:43,666 INFO [M:0;7ae73638d47e:36511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/678c9ca8603141709df6937cfe362b59 2024-12-02T22:16:43,686 DEBUG [M:0;7ae73638d47e:36511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97f3791f0d3e4effbdb4d5963528c1f0 is 2479, key is \x00\x00\x00\x00\x00\x00\x00^/proc:d/1733177712381/Put/seqid=0 2024-12-02T22:16:43,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742541_1717 (size=46751) 2024-12-02T22:16:44,089 INFO [M:0;7ae73638d47e:36511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=780.93 KB at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97f3791f0d3e4effbdb4d5963528c1f0 2024-12-02T22:16:44,093 INFO [M:0;7ae73638d47e:36511 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97f3791f0d3e4effbdb4d5963528c1f0 2024-12-02T22:16:44,107 DEBUG [M:0;7ae73638d47e:36511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05f132cb9e3c4a559d6f77c90367b6be is 69, key is 7ae73638d47e,41753,1733177627056/rs:state/1733177628151/Put/seqid=0 2024-12-02T22:16:44,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073742542_1718 (size=5156) 2024-12-02T22:16:44,367 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T22:16:44,511 INFO [M:0;7ae73638d47e:36511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05f132cb9e3c4a559d6f77c90367b6be 2024-12-02T22:16:44,514 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/678c9ca8603141709df6937cfe362b59 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/678c9ca8603141709df6937cfe362b59 2024-12-02T22:16:44,517 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/678c9ca8603141709df6937cfe362b59, entries=8, sequenceid=2166, filesize=5.5 K 2024-12-02T22:16:44,517 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97f3791f0d3e4effbdb4d5963528c1f0 as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97f3791f0d3e4effbdb4d5963528c1f0 2024-12-02T22:16:44,519 INFO [M:0;7ae73638d47e:36511 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97f3791f0d3e4effbdb4d5963528c1f0 2024-12-02T22:16:44,519 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97f3791f0d3e4effbdb4d5963528c1f0, entries=178, sequenceid=2166, filesize=45.7 K 2024-12-02T22:16:44,520 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05f132cb9e3c4a559d6f77c90367b6be as hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05f132cb9e3c4a559d6f77c90367b6be 2024-12-02T22:16:44,522 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34911/user/jenkins/test-data/51d11872-6298-42de-a9cf-6069fdcec66c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05f132cb9e3c4a559d6f77c90367b6be, entries=1, sequenceid=2166, filesize=5.0 K 2024-12-02T22:16:44,523 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(3140): Finished flush of dataSize ~781.49 KB/800241, heapSize ~957.09 KB/980064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2166, compaction requested=false 2024-12-02T22:16:44,524 INFO [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T22:16:44,525 DEBUG [M:0;7ae73638d47e:36511 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733177803248Disabling compacts and flushes for region at 1733177803248Disabling writes for close at 1733177803248Obtaining lock to block concurrent updates at 1733177803248Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733177803248Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=800241, getHeapSize=980304, getOffHeapSize=0, getCellsCount=2369 at 1733177803248Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733177803249 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733177803249Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733177803263 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733177803263Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733177803670 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733177803685 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733177803685Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733177804093 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733177804107 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733177804107Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a1b191a: reopening flushed file at 1733177804513 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4be3646f: reopening flushed file at 1733177804517 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a5d7bd: reopening flushed file at 1733177804519 (+2 ms)Finished flush of dataSize ~781.49 KB/800241, heapSize ~957.09 KB/980064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2166, compaction requested=false at 1733177804523 (+4 ms)Writing region close event to WAL at 1733177804524 (+1 ms)Closed at 1733177804524 2024-12-02T22:16:44,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46173 is added to blk_1073741830_1006 (size=942262) 2024-12-02T22:16:44,527 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T22:16:44,527 INFO [M:0;7ae73638d47e:36511 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T22:16:44,527 INFO [M:0;7ae73638d47e:36511 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36511 2024-12-02T22:16:44,528 INFO [M:0;7ae73638d47e:36511 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T22:16:44,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:16:44,630 INFO [M:0;7ae73638d47e:36511 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T22:16:44,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36511-0x1009d0dd6140000, quorum=127.0.0.1:58452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T22:16:44,635 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T22:16:44,638 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:16:44,638 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:16:44,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:16:44,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.log.dir/,STOPPED} 2024-12-02T22:16:44,641 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T22:16:44,641 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T22:16:44,641 WARN [BP-1855264631-172.17.0.2-1733177623427 heartbeating to localhost/127.0.0.1:34911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T22:16:44,641 WARN [BP-1855264631-172.17.0.2-1733177623427 heartbeating to localhost/127.0.0.1:34911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1855264631-172.17.0.2-1733177623427 (Datanode Uuid 822023b4-62d7-49d7-a9ba-af97b603c81e) service to localhost/127.0.0.1:34911 2024-12-02T22:16:44,643 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data1/current/BP-1855264631-172.17.0.2-1733177623427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:16:44,643 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/cluster_23e30b23-8414-97e3-6f9f-1c14dbf35b85/data/data2/current/BP-1855264631-172.17.0.2-1733177623427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T22:16:44,644 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T22:16:44,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T22:16:44,651 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T22:16:44,651 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T22:16:44,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T22:16:44,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/811174af-4a50-7be1-d0ac-c3ea213c7dc3/hadoop.log.dir/,STOPPED} 2024-12-02T22:16:44,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T22:16:44,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down